moonq/q-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of bitbucket.org:MoonQ/q-tools

Browse Source
This commit is contained in:
Ville Rantanen
2021-04-09 13:01:22 +03:00
parent 5c44846b10 525c93a106
commit 900f54e3ef
23 changed files with 276 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
tsv/lib/tsvhead
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2015 Ville Rantanen
#
@@ -17,13 +17,14 @@
#
'''simple head for tsv/csv files.'''
from __future__ import print_function
__author__ = "Ville Rantanen <ville.q.rantanen@gmail.com>"
__version__ = "0.1"
import sys,os,argparse
from argparse import ArgumentParser
from argparse import ArgumentParser
def setup_options():
''' Create command line options '''
@@ -31,7 +32,7 @@ def setup_options():
Simple implementation of head that keeps the header row.
'''
parser=ArgumentParser(description=usage,
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="\n".join(["Version: "+__version__,__author__]))
@@ -60,7 +61,7 @@ def behead(fileob,opts):
head_ordinary(fileob, opts.lines)
else:
head_allbutlast(fileob, opts.lines)
def head_ordinary(fileob, lines):
for i,row in enumerate(fileob):
if i>lines-1:
@@ -82,12 +83,13 @@ def main():
behead(sys.stdin, opts)
for fi in opts.file:
behead(open(fi,'r'), opts)
except IOError as (n,e):
except IOError as xxx_todo_changeme:
(n,e) = xxx_todo_changeme.args
if n==32:
pass
else:
import traceback
print traceback.format_exc()
print(traceback.format_exc())
except KeyboardInterrupt:
pass

14
tsv/lib/tsvtail
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2015 Ville Rantanen
#
@@ -17,13 +17,14 @@
#
'''simple tail for tsv/csv files.'''
from __future__ import print_function
__author__ = "Ville Rantanen <ville.q.rantanen@gmail.com>"
__version__ = "0.1"
import sys,os,argparse
from argparse import ArgumentParser
from argparse import ArgumentParser
def setup_options():
''' Create command line options '''
@@ -31,7 +32,7 @@ def setup_options():
simple implementation of tail, keeping the header row
'''
parser=ArgumentParser(description=usage,
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="\n".join(["Version: "+__version__,__author__]))
@@ -64,7 +65,7 @@ def tail(fileob,opts):
tail_ordinary(fileob, opts.lines)
else:
tail_allbutfirst(fileob, -opts.lines)
def tail_allbutfirst(fileob, lines):
for i,row in enumerate(fileob):
if i<lines-1:
@@ -86,12 +87,13 @@ def main():
tail(sys.stdin, opts)
for fi in opts.file:
tail(open(fi,'r'), opts)
except IOError as (n,e):
except IOError as xxx_todo_changeme:
(n,e) = xxx_todo_changeme.args
if n==32:
pass
else:
import traceback
print traceback.format_exc()
print(traceback.format_exc())
except KeyboardInterrupt:
pass

24
tsv/sc2tsv.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2017 Ville Rantanen
#
@@ -23,7 +23,7 @@ __author__ = "Ville Rantanen"
__version__ = "0.1"
import sys
from argparse import ArgumentParser
from argparse import ArgumentParser
import unicodedata, re
import subprocess
@@ -46,12 +46,12 @@ def which(program):
class SCReader:
""" Class for reading SC files.
"""
"""
def __init__(self,data):
self.data=data.split('\n')
self.parserre=re.compile('.* ([A-Z]+)([0-9]+) = (.*)')
def _parse_row(self,string):
col=None
row=None
@@ -62,14 +62,14 @@ class SCReader:
row=self.try_int(m.group(2))
content=m.group(3)
return col,row,content
def try_int(self,string):
try:
return int(string)
except:
return string
def alpha_to_column(self,alpha):
''' Returns a column number from spreadsheet column alphabet '''
n=0
@@ -78,7 +78,7 @@ class SCReader:
o+=(ord(char.upper())-64)*(26**n)
n+=1
return int(o-1)
def next(self):
''' Returns the next row in the table, three items: column, row, content'''
return self._parse_row(self.reader.next())
@@ -89,7 +89,7 @@ class SCReader:
def setup_options():
''' Setup the command line options '''
parser=ArgumentParser()
parser.add_argument("-v","--version",action='version', version=__version__)
parser.add_argument("-d",type=str,dest="delimiter",default="\t",
@@ -97,8 +97,8 @@ def setup_options():
parser.add_argument("sc",type=str,action="store",
help="SC file to convert")
options=parser.parse_args()
return options
return options
def tsv_write(screader,fileout,delim):
''' writes a TSV from SCReader iterator '''
content=[]

49
tsv/tsv2sc.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2017 Ville Rantanen
#
@@ -17,14 +17,21 @@
#
'''TSV 2 SC convert.'''
from __future__ import division
from __future__ import print_function
from builtins import chr
from builtins import str
from builtins import range
from builtins import object
from past.utils import old_div
__author__ = "Ville Rantanen"
__version__ = "0.2"
import sys,os
import csv
from argparse import ArgumentParser
from argparse import ArgumentParser
import unicodedata, re
import subprocess
@@ -45,10 +52,10 @@ def which(program):
return None
class SCWriter:
class SCWriter(object):
""" Class for writing SC files.
"""
"""
def __init__(self,file,alignright,float_precision):
self.file=file
self.row=0
@@ -61,9 +68,9 @@ class SCWriter:
self.alignstring="rightstring"
else:
self.alignstring="leftstring"
def parse_row(self,string):
self.col=0
for el in row:
self.write_element(self.row,self.col,el)
@@ -78,30 +85,30 @@ class SCWriter:
self.col_lengths[self.col]=max(len(el_str)+1,self.col_lengths[self.col])
self.col+=1
self.row+=1
def write_element(self,row,col,content):
colalpha=self.column_to_alpha(col)
content=content.strip('"')
if self.is_num(content):
self.file.write('let '+colalpha+str(row)+' = ' + str(self.to_num(content))+'\n')
else:
self.file.write(self.alignstring+' '+colalpha+str(row)+' = "' + content + '"\n')
def column_to_alpha(self,column):
''' Returns a column alphabet from column number '''
o=chr(column+64+1)
if column>25:
return self.column_to_alpha((column / 26) -1) + self.column_to_alpha(column % 26);
return self.column_to_alpha((old_div(column, 26)) -1) + self.column_to_alpha(column % 26);
return o
def write_row(self,row):
''' Writes a row as a SC file part '''
self.parse_row(row)
def write_formats(self):
for col in range(len(self.col_lengths)):
precision="0"
if self.col_types[col]=='float':
@@ -117,7 +124,7 @@ class SCWriter:
return True
except:
pass
return False
return False
def is_int(self,string):
try:
@@ -125,7 +132,7 @@ class SCWriter:
return True
except:
pass
return False
return False
def is_num(self,string):
@@ -149,12 +156,12 @@ class SCWriter:
return num
except:
pass
return string
def setup_options():
''' Setup the command line options '''
parser=ArgumentParser()
parser.add_argument("-v","--version",action='version', version=__version__)
parser.add_argument("-f",type=int,dest="precision",default=2,
@@ -166,8 +173,8 @@ def setup_options():
parser.add_argument("tsv",type=str,action="store",
help="TSV file to convert")
options=parser.parse_args()
return options
return options
if not which('sc'):
print('You don\'t seem to have "sc" installed!')
print('sudo apt-get install sc')

73
tsv/tsvedit.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2014 Ville Rantanen
#
@@ -17,14 +17,21 @@
#
'''SC based CSV editor.'''
from __future__ import division
from __future__ import print_function
from builtins import chr
from builtins import str
from builtins import range
from past.utils import old_div
from builtins import object
__author__ = "Ville Rantanen"
__version__ = "0.2"
import sys,os
import csv
from argparse import ArgumentParser
from argparse import ArgumentParser
import unicodedata, re
import subprocess
import shutil
@@ -46,14 +53,14 @@ def which(program):
return None
class SCReader:
class SCReader(object):
""" Class for reading SC files.
"""
"""
def __init__(self,fileobject):
self.file=fileobject
self.parserre=re.compile('.* ([A-Z]+)([0-9]+) = (.*)')
def _parse_row(self,string):
col=None
row=None
@@ -64,14 +71,14 @@ class SCReader:
row=self.try_int(m.group(2))
content=m.group(3)
return col,row,content
def try_int(self,string):
try:
return int(string)
except:
return string
def alpha_to_column(self,alpha):
''' Returns a column number from spreadsheet column alphabet '''
n=0
@@ -80,28 +87,28 @@ class SCReader:
o+=(ord(char.upper())-64)*(26**n)
n+=1
return int(o-1)
def next(self):
def __next__(self):
''' Returns the next row in the table, three items: column, row, content'''
return self._parse_row(self.reader.next())
return self._parse_row(next(self.reader))
def __iter__(self):
for row in self.file:
yield self._parse_row(row)
class SCWriter:
class SCWriter(object):
""" Class for writing SC files.
"""
"""
def __init__(self,fileobject):
self.file=fileobject
self.row=0
self.col=0
self.col_lengths=[]
def parse_row(self,string):
self.col=0
for el in row:
self.write_element(self.row,self.col,el)
@@ -111,34 +118,34 @@ class SCWriter:
self.col_lengths[self.col]=max(len(el),self.col_lengths[self.col])
self.col+=1
self.row+=1
def write_element(self,row,col,content):
colalpha=self.column_to_alpha(col)
content=content.strip('"')
if self.is_num(content):
self.file.write('let '+colalpha+str(row)+' = ' + str(self.to_num(content))+'\n')
else:
self.file.write('rightstring '+colalpha+str(row)+' = "' + content + '"\n')
def column_to_alpha(self,column):
''' Returns a column alphabet from column number '''
o=chr(column+64+1)
if column>25:
return self.column_to_alpha((column / 26) -1) + self.column_to_alpha(column % 26);
return self.column_to_alpha((old_div(column, 26)) -1) + self.column_to_alpha(column % 26);
return o
def write_row(self,row):
''' Writes a row as a SC file part '''
self.parse_row(row)
def write_formats(self):
for col in range(len(self.col_lengths)):
self.file.write('format '+self.column_to_alpha(col)+' '+str(self.col_lengths[col])+' 2 0\n')
def is_num(self,string):
''' returns the True if string can be converted to number safely '''
try:
@@ -152,7 +159,7 @@ class SCWriter:
return True
except:
pass
return False
def to_num(self,string):
@@ -168,12 +175,12 @@ class SCWriter:
return num
except:
pass
return string
def setup_options():
''' Setup the command line options '''
parser=ArgumentParser()
parser.add_argument("-v","--version",action='version', version=__version__)
parser.add_argument("-b",action="store_false",dest="backup",default=True,
@@ -185,8 +192,8 @@ def setup_options():
parser.add_argument("tsv",type=str,action="store",
help="TSV file to edit")
options=parser.parse_args()
return options
return options
def csv_write(screader,fileout):
''' writes a CSV from SCReader iterator '''
content=[]

50
tsv/tsvkit.sh
View File

@@ -7,7 +7,7 @@ function c2t {
# Convert comma separated stream in to tab separated stream
# Usage: echo "foo,bar" | c2t
python -c 'import sys,csv
python3 -c 'import sys,csv
try:
csv.writer(sys.stdout, dialect=csv.excel_tab, lineterminator="\n").writerows(csv.reader(sys.stdin, dialect=csv.excel))
except IOError:
@@ -15,15 +15,15 @@ except IOError:
}
function header {
# Print only the first line of input
# Print only the first line of input
# Usage: header file.csv
# Usage: cat file.csv | header
# Usage: cat file.csv | header
head -n 1 "$@"
}
function noheader {
# Strip first row of input
# Strip first row of input
# Usage: noheader file.csv
# Usage: cat file.csv | noheader
@@ -33,7 +33,7 @@ function noheader {
function tsvecho {
# Echo with tab separated values, quoted
# Usage: tsvecho value1 value2 "some value" > header.csv
# Usage: echo value1 value2 | tsvecho
# Usage: echo value1 value2 | tsvecho
local HEAD
[[ -t 0 ]] && {
@@ -48,7 +48,7 @@ function tsvstrip {
# Strip tsv of quotes
# Usage: cat file.csv | tsvstrip
python -c 'import sys,csv
python3 -c 'import sys,csv
try:
csv.writer(sys.stdout, dialect=csv.excel_tab, quoting=csv.QUOTE_NONE).writerows(csv.reader(sys.stdin, dialect=csv.excel_tab))
except IOError:
@@ -59,7 +59,7 @@ function tsvtranspose {
# Transpose a tsv file
# Usage: cat file.csv | tsvtranspose
python -c 'import sys,csv
python3 -c 'import sys,csv
try:
csv.writer(sys.stdout, dialect=csv.excel_tab, quoting=csv.QUOTE_NONE).writerows(map(None,*csv.reader(sys.stdin, dialect=csv.excel_tab)))
except IOError:
@@ -71,9 +71,9 @@ function tsvhead {
# Usage: cat file | tsvhead -n 30
if [ -t 0 ]; then
python "${TSVDIR}"/lib/tsvhead "$@"
python3 "${TSVDIR}"/lib/tsvhead "$@"
else
cat - | python "${TSVDIR}"/lib/tsvhead "$@"
cat - | python3 "${TSVDIR}"/lib/tsvhead "$@"
fi
}
@@ -82,9 +82,9 @@ function tsvtail {
# Usage: cat file | tsvtail -n 30
if [ -t 0 ]; then
python "${TSVDIR}"/lib/tsvtail "$@"
python3 "${TSVDIR}"/lib/tsvtail "$@"
else
cat - | python "${TSVDIR}"/lib/tsvtail "$@"
cat - | python3 "${TSVDIR}"/lib/tsvtail "$@"
fi
}
@@ -94,7 +94,7 @@ function tsvcut {
# csvcut with tab-delimited dialect, see original script for options
# Usage: tsvcut -c Col1,Col3 input1.tsv
csvcut -t "$@" | c2t
}
@@ -102,7 +102,7 @@ function tsvformat {
# csvformat with tab-delimited dialect, see original script for options
# Usage: tsvformat -c Col2 -m searchString input1.tsv
csvformat -t -T "$@"
}
@@ -110,7 +110,7 @@ function tsvgrep {
# csvgrep with tab-delimited dialect, see original script for options
# Usage: tsvgrep -c Col2 -m searchString input1.tsv
csvgrep -t "$@" | c2t
}
@@ -118,7 +118,7 @@ function tsvjoin {
# csvjoin with tab-delimited dialect, see original script for options
# Usage: tsvjoin -c 1,1 input1.tsv input2.tsv
csvjoin -t "$@" | c2t
}
@@ -126,7 +126,7 @@ function tsvlook {
# csvlook with tab-delimited dialect, see original script for options
# Usage: tsvlook file1.tsv
csvlook -t "$@"
}
@@ -147,8 +147,8 @@ function tsvquery {
When defining a database with -d it is kept, and can be
inserted with more data later. Otherwise the DB is created in /tmp/
and deleted afterwards.
If not using name=data.tsv syntax, tables are named tsv1, tsv2...
Note: You have to give an SQL query. If you just want to
If not using name=data.tsv syntax, tables are named tsv1, tsv2...
Note: You have to give an SQL query. If you just want to
populate a database, add " " as an empty query.
'
return 0
@@ -167,7 +167,7 @@ function tsvquery {
# Add table with unique numbering
local OLDTBLS=$( sqlite3 "$DBTEMP" ".tables" )
local TBLNO=1
while :
while :
do echo $OLDTBLS | grep tsv$TBLNO > /dev/null || break
TBLNO=$(( $TBLNO + 1 ))
done
@@ -178,7 +178,7 @@ function tsvquery {
local FIL
TBL=$( echo ${!i} | sed 's,=.*,,' )
FIL=$( echo ${!i} | sed "s,^$TBL=,," )
[ -f "$FIL" ] && {
[ -f "$FIL" ] && {
cat "$FIL" | csvsql -t --db "sqlite:///$DBTEMP" --insert --table "$TBL"
} || {
echo File "${!i}" not found
@@ -191,7 +191,7 @@ function tsvquery {
local EC=$?
# remove DB if using temporary
[ -z "$j" ] && {
rm -f "$DBTEMP"
rm -f "$DBTEMP"
}
return $EC
}
@@ -200,7 +200,7 @@ function tsvsort {
# csvsort with tab-delimited dialect, see original script for options
# Usage: tsvsort -c Col3 input.tsv
csvsort -t "$@" | c2t
}
@@ -208,7 +208,7 @@ function tsvstack {
# csvstack with tab-delimited dialect, see original script for options
# Usage: tsvstack file1.tsv file2.tsv
csvstack -t "$@" | c2t
}
@@ -240,9 +240,9 @@ function tsvfold {
function tsvdims {
# Print dimensions of a TSV
# Usage: tsvdims file.txt
# Usage: cat file.txt | tsvdims
# Usage: cat file.txt | tsvdims
python -c 'import sys,csv
python3 -c 'import sys,csv
if sys.argv[1]=="":
input=sys.stdin
else:

38
tsv/tsvplot.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2016 Ville Rantanen
#
@@ -16,8 +16,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import print_function
from __future__ import division
from past.utils import old_div
import sys, os, math, csv, subprocess
from argparse import ArgumentParser
from argparse import ArgumentParser
'''Plot CSV files with GNUPLOT.'''
__author__ = "Ville Rantanen"
@@ -29,7 +31,7 @@ def get_data_scatter(opts):
try:
for row in reader:
data.append(",".join([row[c] for c in opts.columns]))
except KeyError,ME:
except KeyError as ME:
print("Column {} not found.".format(ME),file=sys.stderr)
print("Columns in the file:"+",".join(reader.fieldnames),file=sys.stderr)
sys.exit(1)
@@ -45,12 +47,12 @@ def get_data_histogram(opts):
data.append(row[opts.columns[0]])
except ValueError:
continue
except KeyError,ME:
except KeyError as ME:
print("Column {} not found.".format(ME),file=sys.stderr)
print("Columns in the file:"+",".join(reader.fieldnames),file=sys.stderr)
sys.exit(1)
return "\n".join(data)
def get_stats(opts):
reader=csv.DictReader(open(os.path.join(*opts.csv),'rb'), dialect=csv.excel_tab, lineterminator="\n")
values_min=float('nan')
@@ -65,13 +67,13 @@ def get_stats(opts):
values_n+=1
values_min=min(f,values_min)
values_max=max(f,values_max)
except KeyError,ME:
except KeyError as ME:
print("Column {} not found.".format(ME),file=sys.stderr)
print("Columns in the file:"+",".join(reader.fieldnames),file=sys.stderr)
sys.exit(1)
return (values_min,values_max,values_n)
def get_plotterm(opts):
def get_plotterm(opts):
if opts.X11:
plotterm=""
else:
@@ -95,9 +97,9 @@ def get_histogram_template(opts):
(height,width)=termsize()
if opts.width:
width=opts.width
bins=min(int(float(width)/4),bins)
bins=min(int(old_div(float(width),4)),bins)
template='''{}
set datafile separator ",";
set datafile separator ",";
set xlabel '{}';
Min = {};
Max = {};
@@ -119,8 +121,8 @@ plot '<cat' u (bin($1)):(1.0) smooth freq title '' {};
def get_scatterplot_template(opts):
plotterm=get_plotterm(opts)
template = '''{}
set datafile separator ",";
set xlabel '{}';
set datafile separator ",";
set xlabel '{}';
set ylabel '{}';
{}
plot '<cat' using 1:2 title '' {};'''.format(
@@ -131,7 +133,7 @@ plot '<cat' using 1:2 title '' {};'''.format(
opts.style
)
return template
def run_gnuplot(opts, template, get_data):
p = subprocess.Popen(['gnuplot','-p','-e',template], stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
comm=p.communicate(input=get_data(opts))
@@ -163,7 +165,7 @@ def which(program):
def setup_options():
''' Setup the command line options '''
parser=ArgumentParser(description="Plot CSV columns to 2D scatter plot, or a histogram. Uses GNUPLOT command line utility to draw plots.")
parser.add_argument("-v","--version",action='version', version=__version__)
parser.add_argument("-c",action="store",dest="columns",type=str,required=True,
@@ -175,7 +177,7 @@ def setup_options():
parser.add_argument("-p",action="store",dest="pre",default="",
help="Gnuplot commands to execute before plotting.")
parser.add_argument("-s",action="store",dest="style",default=None,
help="Plotting style, e.g. 'with lines'. Defaults to points for 2D plot, boxes for histogram.")
help="Plotting style, e.g. 'with lines'. Defaults to points for 2D plot, boxes for histogram.")
parser.add_argument("--width",action="store",type=int,dest="width",default=False,
help="Console width. Default: fullscreen")
parser.add_argument("--height",action="store",type=int,dest="height",default=False,
@@ -184,7 +186,7 @@ def setup_options():
help="Show plot in X11 window, instead of console.")
parser.add_argument("-D",action="store_true",dest="debug",default=False,
help="Print commands before execution")
parser.add_argument("csv",type=str,action="store",
help="CSV file to plot")
options=parser.parse_args()
@@ -197,7 +199,7 @@ def setup_options():
options.style="with boxes;"
else:
options.style="pt '*'"
return options
return options
def termsize():
try:
@@ -205,7 +207,7 @@ def termsize():
except:
(rows,columns)=(25,80)
return (int(rows),int(columns))
if not which('gnuplot'):
print('You don\'t seem to have "gnuplot" installed!')
sys.exit(1)