moonq/q-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of bitbucket.org:MoonQ/q-tools

Browse Source
This commit is contained in:
Ville Rantanen
2021-04-09 13:01:22 +03:00
parent 5c44846b10 525c93a106
commit 900f54e3ef
23 changed files with 276 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -22,7 +22,7 @@ Many of the tools use common unix tools. Some extra tools are required. Here are
the installation commands.
```
sudo apt-get install git sqlite3 python python-magic python-pip python-gtk2 sc \
sudo apt-get install git sqlite3 python3 python3-magic python3-pip python3-gtk2 sc \
gnuplot wget vim rsync curl pv
pip2 install csvkit ncsv

2
bin/ansi-codes
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import sys,os
sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__),

2
bin/clockcurses
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import curses
import os,sys
import time,datetime

2
files/archive-files
View File

@@ -19,7 +19,7 @@ function listfiles() {
}
function count_size() {
cat - | python -c "import sys
cat - | python3 -c "import sys
def sizeof_fmt(num, suffix='B'):
for unit in ['','K','M','G','T','P','E','Z']:
if num < 1024.0:

2
files/archive-subfolders
View File

@@ -22,7 +22,7 @@ function listfolders() {
}
function count_size() {
cat - | python -c "import sys
cat - | python3 -c "import sys
def sizeof_fmt(num, suffix='B'):
for unit in ['','K','M','G','T','P','E','Z']:
if num < 1024.0:

2
files/chr2pix
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import sys
import scipy.misc
import numpy as n

26
files/file_list.py
View File

@@ -1,5 +1,12 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: latin-1 -*-
from __future__ import division
from __future__ import print_function
from future import standard_library
standard_library.install_aliases()
from builtins import zip
from builtins import str
from past.utils import old_div
import sys
import os
import re
@@ -8,7 +15,8 @@ import subprocess
import hashlib
import magic
from argparse import ArgumentParser
import ConfigParser,StringIO,io
import configparser
import io
import datetime
SQLFILE='list_of_files.sqlite'
@@ -67,7 +75,7 @@ def setup_options():
BADDIRS.extend(options.exclude)
if options.duplicate:
options.add=not options.add
options.startpath=unicode(options.startpath, "UTF-8")
options.sqlpath=os.path.dirname(os.path.realpath(options.sqlfile))
return options
@@ -78,7 +86,7 @@ def add_recurse(options):
db=conn.cursor()
prev_path_len=0
for path,dirs,files in os.walk(options.startpath,followlinks=options.symlinks):
sys.stdout.write(("\r%s%s"%(filename_join(path,".",options),(prev_path_len-len(path))*' ')).encode('utf-8'))
sys.stdout.write(("\r%s%s"%(filename_join(path,".",options),(prev_path_len-len(path))*' ')))
prev_path_len=len(path)
dirs=clean_dirs(dirs)
dirs.sort()
@@ -218,11 +226,11 @@ def createdb(options):
object TEXT)')
conn.commit()
config = ConfigParser.RawConfigParser()
config = configparser.RawConfigParser()
config.add_section("General")
config.set("General","Relative",str(options.relative))
config.set("General","FullFile",str(options.fullfile))
store=StringIO.StringIO()
store=io.StringIO()
config.write(store)
db.execute("INSERT INTO config (object) values (?)",(store.getvalue(),))
conn.commit()
@@ -337,7 +345,7 @@ def get_md5(filename,fullfile=False):
anim_i=0
anim_len=len(ANIM)
block_size=2**24
percents_per_block=100/(float(fsize)/block_size)
percents_per_block=old_div(100,(old_div(float(fsize),block_size)))
md5 = hashlib.md5()
with open(filename,'rb') as f:
for chunk in iter(lambda: f.read(block_size), b''):
@@ -425,7 +433,7 @@ def humanize_size(size,precision=1):
defPrecision=0
while size > 1024:
suffixIndex += 1 #increment the index of the suffix
size = size/1024.0 #apply the division
size = old_div(size,1024.0) #apply the division
defPrecision=precision
return "%.*f%s"%(defPrecision,size,suffixes[suffixIndex])
@@ -510,7 +518,7 @@ def stored_options(options):
store=""
for row in db:
store+=row[0]+'\n'
config = ConfigParser.RawConfigParser()
config = configparser.RawConfigParser()
config.readfp(io.BytesIO(store))
options.relative=config.getboolean("General","Relative")
options.fullfile=config.getboolean("General","FullFile")

2
files/file_version.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import os, argparse, sys
from shutil import copyfile, copytree
from datetime import datetime

2
files/image_list_tagger.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import sys,os
from Tkinter import *
from PIL import Image,ImageTk, ImageDraw

7
files/rsync-queue
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# coding=UTF-8
from __future__ import print_function
import sys
import os,time,datetime
import sqlite3
@@ -105,9 +106,9 @@ def list_URLs(options):
db.execute("SELECT * FROM list ORDER BY id")
else:
db.execute("SELECT * FROM list WHERE exitcode > 0 ORDER BY id")
print "EC\tSRC\tTGT"
print("EC\tSRC\tTGT")
for row in db:
print "%s\t%s\t%s" % (row[3],row[1],row[2])
print("%s\t%s\t%s" % (row[3],row[1],row[2]))
return
def start_sync(options):

2
files/sponge
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import sys
from argparse import ArgumentParser

2
qcd_function
View File

@@ -13,7 +13,7 @@ function gcd() {
darwin*) QCDPATH=$( dirname $( realpath ${BASH_SOURCE[0]} ) ) ;;
*) QCDPATH=$( dirname $( readlink -f ${BASH_SOURCE[0]} ) ) ;;
esac
cdto=$( python "$QCDPATH"/files/gcd-findmatch.py "$@" )
cdto=$( python3 "$QCDPATH"/files/gcd-findmatch.py "$@" )
[[ -z "$cdto" ]] && return
\cd "$cdto"
} # gcd ends

2
skel/dispchooser
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import os
import sys

8
tsv/lib/tsvhead
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2015 Ville Rantanen
#
@@ -17,6 +17,7 @@
#
'''simple head for tsv/csv files.'''
from __future__ import print_function
__author__ = "Ville Rantanen <ville.q.rantanen@gmail.com>"
@@ -82,12 +83,13 @@ def main():
behead(sys.stdin, opts)
for fi in opts.file:
behead(open(fi,'r'), opts)
except IOError as (n,e):
except IOError as xxx_todo_changeme:
(n,e) = xxx_todo_changeme.args
if n==32:
pass
else:
import traceback
print traceback.format_exc()
print(traceback.format_exc())
except KeyboardInterrupt:
pass

8
tsv/lib/tsvtail
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2015 Ville Rantanen
#
@@ -17,6 +17,7 @@
#
'''simple tail for tsv/csv files.'''
from __future__ import print_function
__author__ = "Ville Rantanen <ville.q.rantanen@gmail.com>"
@@ -86,12 +87,13 @@ def main():
tail(sys.stdin, opts)
for fi in opts.file:
tail(open(fi,'r'), opts)
except IOError as (n,e):
except IOError as xxx_todo_changeme:
(n,e) = xxx_todo_changeme.args
if n==32:
pass
else:
import traceback
print traceback.format_exc()
print(traceback.format_exc())
except KeyboardInterrupt:
pass

2
tsv/sc2tsv.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2017 Ville Rantanen
#

13
tsv/tsv2sc.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2017 Ville Rantanen
#
@@ -17,7 +17,14 @@
#
'''TSV 2 SC convert.'''
from __future__ import division
from __future__ import print_function
from builtins import chr
from builtins import str
from builtins import range
from builtins import object
from past.utils import old_div
__author__ = "Ville Rantanen"
__version__ = "0.2"
@@ -45,7 +52,7 @@ def which(program):
return None
class SCWriter:
class SCWriter(object):
""" Class for writing SC files.
"""
@@ -93,7 +100,7 @@ class SCWriter:
''' Returns a column alphabet from column number '''
o=chr(column+64+1)
if column>25:
return self.column_to_alpha((column / 26) -1) + self.column_to_alpha(column % 26);
return self.column_to_alpha((old_div(column, 26)) -1) + self.column_to_alpha(column % 26);
return o
def write_row(self,row):

19
tsv/tsvedit.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2014 Ville Rantanen
#
@@ -17,7 +17,14 @@
#
'''SC based CSV editor.'''
from __future__ import division
from __future__ import print_function
from builtins import chr
from builtins import str
from builtins import range
from past.utils import old_div
from builtins import object
__author__ = "Ville Rantanen"
__version__ = "0.2"
@@ -46,7 +53,7 @@ def which(program):
return None
class SCReader:
class SCReader(object):
""" Class for reading SC files.
"""
@@ -82,15 +89,15 @@ class SCReader:
return int(o-1)
def next(self):
def __next__(self):
''' Returns the next row in the table, three items: column, row, content'''
return self._parse_row(self.reader.next())
return self._parse_row(next(self.reader))
def __iter__(self):
for row in self.file:
yield self._parse_row(row)
class SCWriter:
class SCWriter(object):
""" Class for writing SC files.
"""
@@ -127,7 +134,7 @@ class SCWriter:
''' Returns a column alphabet from column number '''
o=chr(column+64+1)
if column>25:
return self.column_to_alpha((column / 26) -1) + self.column_to_alpha(column % 26);
return self.column_to_alpha((old_div(column, 26)) -1) + self.column_to_alpha(column % 26);
return o
def write_row(self,row):

16
tsv/tsvkit.sh
View File

@@ -7,7 +7,7 @@ function c2t {
# Convert comma separated stream in to tab separated stream
# Usage: echo "foo,bar" | c2t
python -c 'import sys,csv
python3 -c 'import sys,csv
try:
csv.writer(sys.stdout, dialect=csv.excel_tab, lineterminator="\n").writerows(csv.reader(sys.stdin, dialect=csv.excel))
except IOError:
@@ -48,7 +48,7 @@ function tsvstrip {
# Strip tsv of quotes
# Usage: cat file.csv | tsvstrip
python -c 'import sys,csv
python3 -c 'import sys,csv
try:
csv.writer(sys.stdout, dialect=csv.excel_tab, quoting=csv.QUOTE_NONE).writerows(csv.reader(sys.stdin, dialect=csv.excel_tab))
except IOError:
@@ -59,7 +59,7 @@ function tsvtranspose {
# Transpose a tsv file
# Usage: cat file.csv | tsvtranspose
python -c 'import sys,csv
python3 -c 'import sys,csv
try:
csv.writer(sys.stdout, dialect=csv.excel_tab, quoting=csv.QUOTE_NONE).writerows(map(None,*csv.reader(sys.stdin, dialect=csv.excel_tab)))
except IOError:
@@ -71,9 +71,9 @@ function tsvhead {
# Usage: cat file | tsvhead -n 30
if [ -t 0 ]; then
python "${TSVDIR}"/lib/tsvhead "$@"
python3 "${TSVDIR}"/lib/tsvhead "$@"
else
cat - | python "${TSVDIR}"/lib/tsvhead "$@"
cat - | python3 "${TSVDIR}"/lib/tsvhead "$@"
fi
}
@@ -82,9 +82,9 @@ function tsvtail {
# Usage: cat file | tsvtail -n 30
if [ -t 0 ]; then
python "${TSVDIR}"/lib/tsvtail "$@"
python3 "${TSVDIR}"/lib/tsvtail "$@"
else
cat - | python "${TSVDIR}"/lib/tsvtail "$@"
cat - | python3 "${TSVDIR}"/lib/tsvtail "$@"
fi
}
@@ -242,7 +242,7 @@ function tsvdims {
# Usage: tsvdims file.txt
# Usage: cat file.txt | tsvdims
python -c 'import sys,csv
python3 -c 'import sys,csv
if sys.argv[1]=="":
input=sys.stdin
else:

12
tsv/tsvplot.py
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#
# Copyright 2016 Ville Rantanen
#
@@ -16,6 +16,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import print_function
from __future__ import division
from past.utils import old_div
import sys, os, math, csv, subprocess
from argparse import ArgumentParser
@@ -29,7 +31,7 @@ def get_data_scatter(opts):
try:
for row in reader:
data.append(",".join([row[c] for c in opts.columns]))
except KeyError,ME:
except KeyError as ME:
print("Column {} not found.".format(ME),file=sys.stderr)
print("Columns in the file:"+",".join(reader.fieldnames),file=sys.stderr)
sys.exit(1)
@@ -45,7 +47,7 @@ def get_data_histogram(opts):
data.append(row[opts.columns[0]])
except ValueError:
continue
except KeyError,ME:
except KeyError as ME:
print("Column {} not found.".format(ME),file=sys.stderr)
print("Columns in the file:"+",".join(reader.fieldnames),file=sys.stderr)
sys.exit(1)
@@ -65,7 +67,7 @@ def get_stats(opts):
values_n+=1
values_min=min(f,values_min)
values_max=max(f,values_max)
except KeyError,ME:
except KeyError as ME:
print("Column {} not found.".format(ME),file=sys.stderr)
print("Columns in the file:"+",".join(reader.fieldnames),file=sys.stderr)
sys.exit(1)
@@ -95,7 +97,7 @@ def get_histogram_template(opts):
(height,width)=termsize()
if opts.width:
width=opts.width
bins=min(int(float(width)/4),bins)
bins=min(int(old_div(float(width),4)),bins)
template='''{}
set datafile separator ",";
set xlabel '{}';

2
vnc/vncdisp
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
import os
import sys

70
web/droopy
View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Droopy (http://stackp.online.fr/droopy)
@@ -52,8 +52,12 @@
# * German translation by Michael.
# 20080408 * First release.
import BaseHTTPServer
import SocketServer
from __future__ import print_function
from future import standard_library
standard_library.install_aliases()
from builtins import str
import http.server
import socketserver
import cgi
import os
import posixpath
@@ -67,7 +71,7 @@ import shutil
import tempfile
import socket
import locale
import urllib
import urllib.request, urllib.parse, urllib.error
LOGO = '''\
_____
@@ -611,7 +615,7 @@ class DroopyFieldStorage(cgi.FieldStorage):
return self.tmpfile
class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
class HTTPUploadHandler(http.server.BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.0'
form_field = 'upfile'
@@ -626,7 +630,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
"""
# -- Parse accept-language header
if not self.headers.has_key("accept-language"):
if "accept-language" not in self.headers:
a = []
else:
a = self.headers["accept-language"]
@@ -642,7 +646,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
# -- Choose the appropriate translation dictionary (default is english)
lang = "en"
for l in a:
if translations.has_key(l):
if l in translations:
lang = l
break
dico = copy.copy(translations[lang])
@@ -665,7 +669,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
if names:
for name in names:
links += '<a href="/%s">%s</a><br/>' % (
urllib.quote(name.encode('utf-8')),
urllib.parse.quote(name.encode('utf-8')),
name)
links = '<div id="files">' + links + '</div>'
dico["files"] = links
@@ -682,7 +686,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_GET(self):
name = self.path.lstrip('/')
name = urllib.unquote(name)
name = urllib.parse.unquote(name)
name = name.decode('utf-8')
if picture != None and self.path == '/__droopy/picture':
@@ -714,7 +718,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
form = DroopyFieldStorage(fp = self.rfile, environ = env);
fileitem = form[self.form_field]
realname=""
if self.form_realname in form.keys():
if self.form_realname in list(form.keys()):
realname = form[self.form_realname].value.decode('utf-8')
if realname=="":
filename = self.basename(fileitem.filename).decode('utf-8')
@@ -757,7 +761,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
else:
self.send_html(self.html("success"))
except Exception, e:
except Exception as e:
self.log_message(repr(e))
self.send_html(self.html("error"))
@@ -792,7 +796,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
if publish_files:
# os.listdir() returns a list of unicode strings when the
# directory is passed as an unicode string itself.
names = [name for name in os.listdir(unicode(directory))
names = [name for name in os.listdir(str(directory))
if os.path.isfile(os.path.join(directory, name))
and not name.startswith(DroopyFieldStorage.TMPPREFIX)]
names.sort()
@@ -802,8 +806,8 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def handle(self):
try:
BaseHTTPServer.BaseHTTPRequestHandler.handle(self)
except socket.error, e:
http.server.BaseHTTPRequestHandler.handle(self)
except socket.error as e:
self.log_message(str(e))
raise Abort()
@@ -811,14 +815,14 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
class Abort(Exception): pass
class ThreadedHTTPServer(SocketServer.ThreadingMixIn,
BaseHTTPServer.HTTPServer):
class ThreadedHTTPServer(socketserver.ThreadingMixIn,
http.server.HTTPServer):
def handle_error(self, request, client_address):
# Override SocketServer.handle_error
exctype = sys.exc_info()[0]
if not exctype is Abort:
BaseHTTPServer.HTTPServer.handle_error(self,request,client_address)
http.server.HTTPServer.handle_error(self,request,client_address)
# -- Options
@@ -867,7 +871,7 @@ def load_options():
parse_args(cmd)
f.close()
return True
except IOError, e:
except IOError as e:
return False
@@ -891,8 +895,8 @@ def parse_args(cmd=None):
"directory=", "help",
"save-config","delete-config",
"dl"])
except Exception, e:
print e
except Exception as e:
print(e)
sys.exit(1)
for o,a in opts:
@@ -912,16 +916,16 @@ def parse_args(cmd=None):
try:
filename = configfile()
os.remove(filename)
print 'Deleted ' + filename
except Exception, e:
print e
print('Deleted ' + filename)
except Exception as e:
print(e)
sys.exit(0)
elif o in ['--dl']:
publish_files = True
elif o in ['-h', '--help']:
print USAGE
print(USAGE)
sys.exit(0)
# port number
@@ -929,7 +933,7 @@ def parse_args(cmd=None):
if args[0:]:
port = int(args[0])
except ValueError:
print args[0], "is not a valid port number"
print(args[0], "is not a valid port number")
sys.exit(1)
@@ -944,25 +948,25 @@ def run():
if __name__ == '__main__':
print LOGO
print(LOGO)
config_found = load_options()
parse_args()
if config_found:
print 'Configuration found in %s' % configfile()
print('Configuration found in %s' % configfile())
else:
print "No configuration file found."
print("No configuration file found.")
if must_save_options:
save_options()
print "Options saved in %s" % configfile()
print("Options saved in %s" % configfile())
print "Files will be uploaded to %s" % directory
print("Files will be uploaded to %s" % directory)
try:
print
print "HTTP server running... Check it out at http://localhost:%d"%port
print()
print("HTTP server running... Check it out at http://localhost:%d"%port)
run()
except KeyboardInterrupt:
print '^C received, shutting down server'
print('^C received, shutting down server')
# some threads may run until they terminate

6
web/ftpserver
View File

@@ -1,8 +1,8 @@
#!/bin/sh
python -c "import pyftpdlib" >/dev/null 2>&1 || {
python3 -c "import pyftpdlib" >/dev/null 2>&1 || {
echo "pyftpdlib is not installed:"
echo " pip install --user pyftpdlib"
echo " pip3 install --user pyftpdlib"
exit
}
python -m pyftpdlib "$@"
python3 -m pyftpdlib "$@"