Merge branch 'master' of bitbucket.org:MoonQ/q-tools

README.md

@@ -22,7 +22,7 @@ Many of the tools use common unix tools. Some extra tools are required. Here are
 the installation commands.
 
 ```
-sudo apt-get install git sqlite3 python python-magic python-pip python-gtk2 sc \
+sudo apt-get install git sqlite3 python3 python3-magic python3-pip python3-gtk2 sc \
   gnuplot wget vim rsync curl pv
 pip2 install csvkit ncsv
 

bin/ansi-codes

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys,os
 sys.path.append(os.path.realpath(os.path.join(os.path.dirname(__file__),

bin/clockcurses

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import curses
 import os,sys
 import time,datetime

files/archive-files

@@ -19,7 +19,7 @@ function listfiles() {
 }
 function count_size() {
 
-    cat - | python -c "import sys
+    cat - | python3 -c "import sys
 def sizeof_fmt(num, suffix='B'):
     for unit in ['','K','M','G','T','P','E','Z']:
         if num < 1024.0:

files/archive-subfolders

@@ -22,7 +22,7 @@ function listfolders() {
 }
 function count_size() {
 
-    cat - | python -c "import sys
+    cat - | python3 -c "import sys
 def sizeof_fmt(num, suffix='B'):
     for unit in ['','K','M','G','T','P','E','Z']:
         if num < 1024.0:

files/chr2pix

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import sys
 import scipy.misc
 import numpy as n

files/file_list.py

@@ -1,5 +1,12 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: latin-1 -*-
+from __future__ import division
+from __future__ import print_function
+from future import standard_library
+standard_library.install_aliases()
+from builtins import zip
+from builtins import str
+from past.utils import old_div
 import sys
 import os
 import re
@@ -8,7 +15,8 @@ import subprocess
 import hashlib
 import magic
 from argparse import ArgumentParser
-import ConfigParser,StringIO,io
+import configparser
+import io
 import datetime
 
 SQLFILE='list_of_files.sqlite'
@@ -67,7 +75,7 @@ def setup_options():
     BADDIRS.extend(options.exclude)
     if options.duplicate:
         options.add=not options.add
-    options.startpath=unicode(options.startpath, "UTF-8")
+
     options.sqlpath=os.path.dirname(os.path.realpath(options.sqlfile))
     return options
 
@@ -78,7 +86,7 @@ def add_recurse(options):
     db=conn.cursor()
     prev_path_len=0
     for path,dirs,files in os.walk(options.startpath,followlinks=options.symlinks):
-        sys.stdout.write(("\r%s%s"%(filename_join(path,".",options),(prev_path_len-len(path))*' ')).encode('utf-8'))
+        sys.stdout.write(("\r%s%s"%(filename_join(path,".",options),(prev_path_len-len(path))*' ')))
         prev_path_len=len(path)
         dirs=clean_dirs(dirs)
         dirs.sort()
@@ -218,11 +226,11 @@ def createdb(options):
                object TEXT)')
     conn.commit()
 
-    config = ConfigParser.RawConfigParser()
+    config = configparser.RawConfigParser()
     config.add_section("General")
     config.set("General","Relative",str(options.relative))
     config.set("General","FullFile",str(options.fullfile))
-    store=StringIO.StringIO()
+    store=io.StringIO()
     config.write(store)
     db.execute("INSERT INTO config (object) values (?)",(store.getvalue(),))
     conn.commit()
@@ -337,7 +345,7 @@ def get_md5(filename,fullfile=False):
         anim_i=0
         anim_len=len(ANIM)
         block_size=2**24
-        percents_per_block=100/(float(fsize)/block_size)
+        percents_per_block=old_div(100,(old_div(float(fsize),block_size)))
         md5 = hashlib.md5()
         with open(filename,'rb') as f:
             for chunk in iter(lambda: f.read(block_size), b''):
@@ -425,7 +433,7 @@ def humanize_size(size,precision=1):
     defPrecision=0
     while size > 1024:
         suffixIndex += 1 #increment the index of the suffix
-        size = size/1024.0 #apply the division
+        size = old_div(size,1024.0) #apply the division
         defPrecision=precision
     return "%.*f%s"%(defPrecision,size,suffixes[suffixIndex])
 
@@ -510,7 +518,7 @@ def stored_options(options):
         store=""
         for row in db:
             store+=row[0]+'\n'
-        config = ConfigParser.RawConfigParser()
+        config = configparser.RawConfigParser()
         config.readfp(io.BytesIO(store))
         options.relative=config.getboolean("General","Relative")
         options.fullfile=config.getboolean("General","FullFile")

files/file_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os, argparse, sys
 from shutil import copyfile, copytree
 from datetime import datetime

files/image_list_tagger.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import sys,os
 from Tkinter import *
 from PIL import Image,ImageTk, ImageDraw

files/rsync-queue

@@ -1,5 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # coding=UTF-8
+from __future__ import print_function
 import sys
 import os,time,datetime
 import sqlite3
@@ -105,9 +106,9 @@ def list_URLs(options):
         db.execute("SELECT * FROM list ORDER BY id")
     else:
         db.execute("SELECT * FROM list WHERE exitcode > 0 ORDER BY id")
-    print "EC\tSRC\tTGT"
+    print("EC\tSRC\tTGT")
     for row in db:
-        print "%s\t%s\t%s" % (row[3],row[1],row[2])
+        print("%s\t%s\t%s" % (row[3],row[1],row[2]))
     return
 
 def start_sync(options):

files/sponge

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys
 from argparse import ArgumentParser

qcd_function

@@ -13,7 +13,7 @@ function gcd() {
         darwin*) QCDPATH=$( dirname $( realpath ${BASH_SOURCE[0]} ) ) ;;
         *)       QCDPATH=$( dirname $( readlink -f ${BASH_SOURCE[0]} ) ) ;;
     esac
-    cdto=$( python "$QCDPATH"/files/gcd-findmatch.py "$@" )
+    cdto=$( python3 "$QCDPATH"/files/gcd-findmatch.py "$@" )
     [[ -z "$cdto" ]] && return
     \cd "$cdto"
 } # gcd ends

skel/dispchooser

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys

tsv/lib/tsvhead

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2015 Ville Rantanen
 #
@@ -17,6 +17,7 @@
 #
 
 '''simple head for tsv/csv files.'''
+from __future__ import print_function
 
 __author__ = "Ville Rantanen <ville.q.rantanen@gmail.com>"
 
@@ -82,12 +83,13 @@ def main():
             behead(sys.stdin, opts)
         for fi in opts.file:
             behead(open(fi,'r'), opts)
-    except IOError as (n,e):
+    except IOError as xxx_todo_changeme:
+        (n,e) = xxx_todo_changeme.args
         if  n==32:
             pass
         else:
             import traceback
-            print traceback.format_exc()
+            print(traceback.format_exc())
     except KeyboardInterrupt:
         pass
 

tsv/lib/tsvtail

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2015 Ville Rantanen
 #
@@ -17,6 +17,7 @@
 #
 
 '''simple tail for tsv/csv files.'''
+from __future__ import print_function
 
 __author__ = "Ville Rantanen <ville.q.rantanen@gmail.com>"
 
@@ -86,12 +87,13 @@ def main():
             tail(sys.stdin, opts)
         for fi in opts.file:
             tail(open(fi,'r'), opts)
-    except IOError as (n,e):
+    except IOError as xxx_todo_changeme:
+        (n,e) = xxx_todo_changeme.args
         if  n==32:
             pass
         else:
             import traceback
-            print traceback.format_exc()
+            print(traceback.format_exc())
     except KeyboardInterrupt:
         pass
 

tsv/sc2tsv.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2017 Ville Rantanen
 #

tsv/tsv2sc.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2017 Ville Rantanen
 #
@@ -17,7 +17,14 @@
 #
 
 '''TSV 2 SC convert.'''
+from __future__ import division
+from __future__ import print_function
 
+from builtins import chr
+from builtins import str
+from builtins import range
+from builtins import object
+from past.utils import old_div
 __author__ = "Ville Rantanen"
 
 __version__ = "0.2"
@@ -45,7 +52,7 @@ def which(program):
 
     return None
 
-class SCWriter:
+class SCWriter(object):
     """ Class for writing SC files.
 
     """
@@ -93,7 +100,7 @@ class SCWriter:
         ''' Returns a column alphabet from column number '''
         o=chr(column+64+1)
         if column>25:
-            return self.column_to_alpha((column / 26) -1) + self.column_to_alpha(column % 26);
+            return self.column_to_alpha((old_div(column, 26)) -1) + self.column_to_alpha(column % 26);
         return o
 
     def write_row(self,row):

tsv/tsvedit.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2014 Ville Rantanen
 #
@@ -17,7 +17,14 @@
 #
 
 '''SC based CSV editor.'''
+from __future__ import division
+from __future__ import print_function
 
+from builtins import chr
+from builtins import str
+from builtins import range
+from past.utils import old_div
+from builtins import object
 __author__ = "Ville Rantanen"
 
 __version__ = "0.2"
@@ -46,7 +53,7 @@ def which(program):
 
     return None
 
-class SCReader:
+class SCReader(object):
     """ Class for reading SC files.
 
     """
@@ -82,15 +89,15 @@ class SCReader:
         return int(o-1)
 
 
-    def next(self):
+    def __next__(self):
         ''' Returns the next row in the table, three items: column, row, content'''
-        return self._parse_row(self.reader.next())
+        return self._parse_row(next(self.reader))
 
     def __iter__(self):
         for row in self.file:
             yield self._parse_row(row)
 
-class SCWriter:
+class SCWriter(object):
     """ Class for writing SC files.
 
     """
@@ -127,7 +134,7 @@ class SCWriter:
         ''' Returns a column alphabet from column number '''
         o=chr(column+64+1)
         if column>25:
-            return self.column_to_alpha((column / 26) -1) + self.column_to_alpha(column % 26);
+            return self.column_to_alpha((old_div(column, 26)) -1) + self.column_to_alpha(column % 26);
         return o
 
     def write_row(self,row):

tsv/tsvkit.sh

@@ -7,7 +7,7 @@ function c2t {
 # Convert comma separated stream in to tab separated stream
 # Usage: echo "foo,bar" | c2t
 
-    python -c 'import sys,csv
+    python3 -c 'import sys,csv
 try:
     csv.writer(sys.stdout, dialect=csv.excel_tab, lineterminator="\n").writerows(csv.reader(sys.stdin, dialect=csv.excel))
 except IOError:
@@ -48,7 +48,7 @@ function tsvstrip {
 # Strip tsv of quotes
 # Usage: cat file.csv | tsvstrip
 
-    python -c 'import sys,csv
+    python3 -c 'import sys,csv
 try:
     csv.writer(sys.stdout, dialect=csv.excel_tab, quoting=csv.QUOTE_NONE).writerows(csv.reader(sys.stdin, dialect=csv.excel_tab))
 except IOError:
@@ -59,7 +59,7 @@ function tsvtranspose {
 # Transpose a tsv file
 # Usage: cat file.csv | tsvtranspose
 
-    python -c 'import sys,csv
+    python3 -c 'import sys,csv
 try:
     csv.writer(sys.stdout, dialect=csv.excel_tab, quoting=csv.QUOTE_NONE).writerows(map(None,*csv.reader(sys.stdin, dialect=csv.excel_tab)))
 except IOError:
@@ -71,9 +71,9 @@ function tsvhead {
 # Usage: cat file | tsvhead -n 30
 
     if [ -t 0 ]; then
-        python "${TSVDIR}"/lib/tsvhead "$@"
+        python3 "${TSVDIR}"/lib/tsvhead "$@"
     else
-        cat - | python "${TSVDIR}"/lib/tsvhead "$@"
+        cat - | python3 "${TSVDIR}"/lib/tsvhead "$@"
     fi
 }
 
@@ -82,9 +82,9 @@ function tsvtail {
 # Usage: cat file | tsvtail -n 30
 
     if [ -t 0 ]; then
-        python "${TSVDIR}"/lib/tsvtail "$@"
+        python3 "${TSVDIR}"/lib/tsvtail "$@"
     else
-        cat - | python "${TSVDIR}"/lib/tsvtail "$@"
+        cat - | python3 "${TSVDIR}"/lib/tsvtail "$@"
     fi
 }
 
@@ -242,7 +242,7 @@ function tsvdims {
 # Usage: tsvdims file.txt
 # Usage: cat file.txt | tsvdims
 
-    python -c 'import sys,csv
+    python3 -c 'import sys,csv
 if sys.argv[1]=="":
    input=sys.stdin
 else:

tsv/tsvplot.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright 2016 Ville Rantanen
 #
@@ -16,6 +16,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 from __future__ import print_function
+from __future__ import division
+from past.utils import old_div
 import sys, os, math, csv, subprocess
 from argparse import ArgumentParser
 
@@ -29,7 +31,7 @@ def get_data_scatter(opts):
     try:
         for row in reader:
             data.append(",".join([row[c] for c in opts.columns]))
-    except KeyError,ME:
+    except KeyError as ME:
         print("Column {} not found.".format(ME),file=sys.stderr)
         print("Columns in the file:"+",".join(reader.fieldnames),file=sys.stderr)
         sys.exit(1)
@@ -45,7 +47,7 @@ def get_data_histogram(opts):
                 data.append(row[opts.columns[0]])
             except ValueError:
                 continue
-    except KeyError,ME:
+    except KeyError as ME:
         print("Column {} not found.".format(ME),file=sys.stderr)
         print("Columns in the file:"+",".join(reader.fieldnames),file=sys.stderr)
         sys.exit(1)
@@ -65,7 +67,7 @@ def get_stats(opts):
             values_n+=1
             values_min=min(f,values_min)
             values_max=max(f,values_max)
-    except KeyError,ME:
+    except KeyError as ME:
         print("Column {} not found.".format(ME),file=sys.stderr)
         print("Columns in the file:"+",".join(reader.fieldnames),file=sys.stderr)
         sys.exit(1)
@@ -95,7 +97,7 @@ def get_histogram_template(opts):
             (height,width)=termsize()
             if opts.width:
                 width=opts.width
-            bins=min(int(float(width)/4),bins)
+            bins=min(int(old_div(float(width),4)),bins)
     template='''{}
 set datafile separator ",";
 set xlabel '{}';

vnc/vncdisp

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os
 import sys

web/droopy

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 # Droopy (http://stackp.online.fr/droopy)
@@ -52,8 +52,12 @@
 #            * German translation by Michael.
 #   20080408 * First release.
 
-import BaseHTTPServer
+from __future__ import print_function
-import SocketServer
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+import http.server
+import socketserver
 import cgi
 import os
 import posixpath
@@ -67,7 +71,7 @@ import shutil
 import tempfile
 import socket
 import locale
-import urllib
+import urllib.request, urllib.parse, urllib.error
 
 LOGO = '''\
  _____
@@ -611,7 +615,7 @@ class DroopyFieldStorage(cgi.FieldStorage):
         return self.tmpfile
 
 
-class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class HTTPUploadHandler(http.server.BaseHTTPRequestHandler):
 
     protocol_version = 'HTTP/1.0'
     form_field = 'upfile'
@@ -626,7 +630,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         """
 
         # -- Parse accept-language header
-        if not self.headers.has_key("accept-language"):
+        if "accept-language" not in self.headers:
             a = []
         else:
             a = self.headers["accept-language"]
@@ -642,7 +646,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         # -- Choose the appropriate translation dictionary (default is english)
         lang = "en"
         for l in a:
-            if translations.has_key(l):
+            if l in translations:
                 lang = l
                 break
         dico = copy.copy(translations[lang])
@@ -665,7 +669,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         if names:
             for name in names:
                 links += '<a href="/%s">%s</a><br/>' % (
-                                urllib.quote(name.encode('utf-8')),
+                                urllib.parse.quote(name.encode('utf-8')),
                                 name)
             links = '<div id="files">' + links + '</div>'
         dico["files"] = links
@@ -682,7 +686,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def do_GET(self):
         name = self.path.lstrip('/')
-        name = urllib.unquote(name)
+        name = urllib.parse.unquote(name)
         name = name.decode('utf-8')
 
         if picture != None and self.path == '/__droopy/picture':
@@ -714,7 +718,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
             form = DroopyFieldStorage(fp = self.rfile, environ = env);
             fileitem = form[self.form_field]
             realname=""
-            if self.form_realname in form.keys():
+            if self.form_realname in list(form.keys()):
                 realname = form[self.form_realname].value.decode('utf-8')
             if realname=="":
                 filename = self.basename(fileitem.filename).decode('utf-8')
@@ -757,7 +761,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
             else:
                 self.send_html(self.html("success"))
 
-        except Exception, e:
+        except Exception as e:
             self.log_message(repr(e))
             self.send_html(self.html("error"))
 
@@ -792,7 +796,7 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
         if publish_files:
             # os.listdir() returns a list of unicode strings when the
             # directory is passed as an unicode string itself.
-            names = [name for name in os.listdir(unicode(directory))
+            names = [name for name in os.listdir(str(directory))
                      if os.path.isfile(os.path.join(directory, name))
                      and not name.startswith(DroopyFieldStorage.TMPPREFIX)]
             names.sort()
@@ -802,8 +806,8 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 
     def handle(self):
         try:
-            BaseHTTPServer.BaseHTTPRequestHandler.handle(self)
+            http.server.BaseHTTPRequestHandler.handle(self)
-        except socket.error, e:
+        except socket.error as e:
             self.log_message(str(e))
             raise Abort()
 
@@ -811,14 +815,14 @@ class HTTPUploadHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 class Abort(Exception): pass
 
 
-class ThreadedHTTPServer(SocketServer.ThreadingMixIn,
+class ThreadedHTTPServer(socketserver.ThreadingMixIn,
-                         BaseHTTPServer.HTTPServer):
+                         http.server.HTTPServer):
 
     def handle_error(self, request, client_address):
         # Override SocketServer.handle_error
         exctype = sys.exc_info()[0]
         if not exctype is Abort:
-            BaseHTTPServer.HTTPServer.handle_error(self,request,client_address)
+            http.server.HTTPServer.handle_error(self,request,client_address)
 
 
 # -- Options
@@ -867,7 +871,7 @@ def load_options():
         parse_args(cmd)
         f.close()
         return True
-    except IOError, e:
+    except IOError as e:
         return False
 
 
@@ -891,8 +895,8 @@ def parse_args(cmd=None):
                                         "directory=", "help",
                                         "save-config","delete-config",
                                         "dl"])
-    except Exception, e:
+    except Exception as e:
-        print e
+        print(e)
         sys.exit(1)
 
     for o,a in opts:
@@ -912,16 +916,16 @@ def parse_args(cmd=None):
             try:
                 filename = configfile()
                 os.remove(filename)
-                print 'Deleted ' + filename
+                print('Deleted ' + filename)
-            except Exception, e:
+            except Exception as e:
-                print e
+                print(e)
             sys.exit(0)
 
         elif o in ['--dl']:
             publish_files = True
 
         elif o in ['-h', '--help']:
-            print USAGE
+            print(USAGE)
             sys.exit(0)
 
     # port number
@@ -929,7 +933,7 @@ def parse_args(cmd=None):
         if args[0:]:
             port = int(args[0])
     except ValueError:
-        print args[0], "is not a valid port number"
+        print(args[0], "is not a valid port number")
         sys.exit(1)
 
 
@@ -944,25 +948,25 @@ def run():
 
 
 if __name__ == '__main__':
-    print LOGO
+    print(LOGO)
 
     config_found = load_options()
     parse_args()
 
     if config_found:
-        print 'Configuration found in %s' % configfile()
+        print('Configuration found in %s' % configfile())
     else:
-        print "No configuration file found."
+        print("No configuration file found.")
 
     if must_save_options:
         save_options()
-        print "Options saved in %s" % configfile()
+        print("Options saved in %s" % configfile())
 
-    print "Files will be uploaded to %s" % directory
+    print("Files will be uploaded to %s" % directory)
     try:
-        print
+        print()
-        print "HTTP server running... Check it out at http://localhost:%d"%port
+        print("HTTP server running... Check it out at http://localhost:%d"%port)
         run()
     except KeyboardInterrupt:
-        print '^C received, shutting down server'
+        print('^C received, shutting down server')
         # some threads may run until they terminate

web/ftpserver

@@ -1,8 +1,8 @@
 #!/bin/sh
 
-python -c "import pyftpdlib" >/dev/null 2>&1 || {
+python3 -c "import pyftpdlib" >/dev/null 2>&1 || {
     echo "pyftpdlib is not installed:"
-    echo "    pip install --user pyftpdlib"
+    echo "    pip3 install --user pyftpdlib"
     exit
 }
-python -m pyftpdlib "$@"
+python3 -m pyftpdlib "$@"