split upload with moar features

code/app.py

@@ -133,8 +133,14 @@ def upload_join_splitted(name, token):
             return "Upload not allowed",400
         if not 'filename' in request.form:
             return "No filename given", 400
+        if not 'parts' in request.form:
+            return "No parts count given", 400
+        try:
+            no_parts = int(request.form['parts'])
+        except:
+            return "Parts not parseable", 400
         parts = []
-        for part in range(500):
+        for part in range(no_parts):
             filename = os.path.join(
                 share['path'],
                 ".%s.part.%03d"%(
@@ -351,42 +357,11 @@ def script_upload(name = None, token = None):
         return share
     if not get_or_none('upload', share) == True:
         return "Upload not allowed",400
-    return """#!/bin/bash
+    return render_template(
-test -n "$1" || {
+        "upload.sh",
-  echo "Add files to upload as argument"
+        name = name,
-  exit 1
+        token = token,
-}
+        rooturl = request.url_root
-CAT=$( which cat )
-which pv &> /dev/null && CAT=$( which pv )
-ROOTURL="%s"
-SHARE="%s"
-TOKEN="%s"
-
-send_file() {
-  $CAT "$file_name" | curl -F "file=@-;filename=${base_name}" ${ROOTURL}upload/${SHARE}/${TOKEN}
-}
-send_folder() {
-  which pv &> /dev/null && printf -v dusize -- "--size %%dk" $( du -s -k "$file_name" | cut -f1 )
-  tar cz "$file_name" | $CAT $dusize - |  curl -F "file=@-;filename=${base_name}.tgz" ${ROOTURL}upload/${SHARE}/${TOKEN}
-}
-
-for file_name in "$@"; do
-  base_name=$( basename "$file_name" )
-  test -f "$file_name" && {
-      printf "Sending file: %%s\n" "$file_name"
-      send_file
-      continue
-  }
-  test -d "$file_name" && {
-      printf "Sending folder: %%s\n" "$file_name"
-      send_folder
-      continue
-  }
-done
-    """%(
-        request.url_root,
-        name,
-        token
     )
 
 
@@ -395,99 +370,40 @@ def script_download(name = None, token = None):
     (ok,share) = get_share(name, token = token)
     if not ok:
         return share
-    files = []
+    commands = []
     for file in iter_folder_files(share['path']):
         status = file_stat(share['path'], file)
-        files.append(status)
+        commands.append('get_file "%s"'%(
-    script = """#!/bin/bash
+            status['url'],
-test "$1" = "-h" && {
+        ))
-  echo "Add argument -f to overwrite files"
+    return render_template(
-  exit 0
+        "download.sh",
-}
+        name = name,
-test "$1" = "-f" && FORCE=1
+        token = token,
-which curl &> /dev/null || {
+        rooturl = request.url_root,
-  echo "curl required"
+        commands = "\n".join(commands)
-  exit 1
-}
-ROOTURL="%s"
-SHARE="%s"
-TOKEN="%s"
-get_file() {
-  WRITE=0
-  FILENAME="$1"
-  test "$FORCE" = "1"   && WRITE=1
-  test -f "${FILENAME}" || WRITE=1
-  test "$WRITE" = "1"   && {
-    echo Downloading ${FILENAME}
-    mkdir -p $( dirname "$FILENAME" )
-    curl "${ROOTURL}download/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}"
-  } || {
-    echo Skipping ${FILENAME}
-  }
-}
-"""%(
-        request.url_root,
-        name,
-        token
     )
 
-    for file in files:
-        script += 'get_file "%s"\n'%(
-            file['url'],
-        )
-    return script
-
 
 @app.route('/script/direct/<name>/<token>', methods=['GET'])
 def script_direct(name = None, token = None):
     (ok,share) = get_share(name, token = token)
     if not ok:
         return share
-    files = []
+    commands = []
     for file in iter_folder_files(share['path']):
         status = file_stat(share['path'], file)
-        status.update({
+        commands.append('get_file "%s" "%s"'%(
-            'token': get_direct_token(share, file)
+            status['url'],
-        })
+            get_direct_token(share, file)
-        files.append(status)
+        ))
-    script = """#!/bin/bash
+    return render_template(
-test "$1" = "-h" && {
+        "download_direct.sh",
-  echo "Add argument -f to overwrite files"
+        name = name,
-  exit 0
+        rooturl = request.url_root,
-}
+        commands = "\n".join(commands)
-test "$1" = "-f" && FORCE=1
-which curl &> /dev/null || {
-  echo "curl required"
-  exit 1
-}
-ROOTURL="%s"
-SHARE="%s"
-get_file() {
-  WRITE=0
-  FILENAME="$1"
-  TOKEN="$2"
-  test "$FORCE" = "1"   && WRITE=1
-  test -f "${FILENAME}" || WRITE=1
-  test "$WRITE" = "1"   && {
-    echo Downloading ${FILENAME}
-    mkdir -p $( dirname "$FILENAME" )
-    curl "${ROOTURL}direct/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}"
-  } || {
-    echo Skipping ${FILENAME}
-  }
-}
-"""%(
-        request.url_root,
-        name,
     )
 
-    for file in files:
-        script += 'get_file "%s" "%s"\n'%(
-            file['url'],
-            file['token'],
-        )
-    return script
-
 
 @app.route('/script/upload_split/<name>/<token>', methods=['GET'])
 def script_upload_split(name = None, token = None):

code/templates/download.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+test "$1" = "-h" && {
+  echo "Add argument -f to overwrite files"
+  exit 0
+}
+test "$1" = "-f" && FORCE=1
+which curl &> /dev/null || {
+  echo "curl required"
+  exit 1
+}
+ROOTURL="{{ rooturl }}"
+SHARE="{{ name }}"
+TOKEN="{{ token }}"
+
+get_file() {
+  WRITE=0
+  FILENAME="$1"
+  test "$FORCE" = "1"   && WRITE=1
+  test -f "${FILENAME}" || WRITE=1
+  test "$WRITE" = "1"   && {
+    echo Downloading ${FILENAME}
+    mkdir -p $( dirname "$FILENAME" )
+    curl "${ROOTURL}download/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}"
+  } || {
+    echo Skipping ${FILENAME}
+  }
+}
+{{ commands }}

code/templates/download_direct.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+test "$1" = "-h" && {
+  echo "Add argument -f to overwrite files"
+  exit 0
+}
+test "$1" = "-f" && FORCE=1
+which curl &> /dev/null || {
+  echo "curl required"
+  exit 1
+}
+ROOTURL="{{ rooturl }}"
+SHARE="{{ name }}"
+
+get_file() {
+  WRITE=0
+  FILENAME="$1"
+  TOKEN="$2"
+  test "$FORCE" = "1"   && WRITE=1
+  test -f "${FILENAME}" || WRITE=1
+  test "$WRITE" = "1"   && {
+    echo Downloading ${FILENAME}
+    mkdir -p $( dirname "$FILENAME" )
+    curl "${ROOTURL}direct/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}"
+  } || {
+    echo Skipping ${FILENAME}
+  }
+}
+{{ commands }}

code/templates/upload.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+test -n "$1" || {
+  echo "Add files to upload as argument"
+  exit 1
+}
+CAT=$( which cat )
+which pv &> /dev/null && CAT=$( which pv )
+ROOTURL="{{ rooturl }}"
+SHARE="{{ name }}"
+TOKEN="{{ token }}"
+
+send_file() {
+  $CAT "$file_name" | curl -F "file=@-;filename=${base_name}" ${ROOTURL}upload/${SHARE}/${TOKEN}
+}
+send_folder() {
+  which pv &> /dev/null && printf -v dusize -- "--size %dk" $( du -s -k "$file_name" | cut -f1 )
+  tar cz "$file_name" | $CAT $dusize - |  curl -F "file=@-;filename=${base_name}.tgz" ${ROOTURL}upload/${SHARE}/${TOKEN}
+}
+
+for file_name in "$@"; do
+  base_name=$( basename "$file_name" )
+  test -f "$file_name" && {
+      printf "Sending file: %s\n" "$file_name"
+      send_file
+      continue
+  }
+  test -d "$file_name" && {
+      printf "Sending folder: %s\n" "$file_name"
+      send_folder
+      continue
+  }
+done

code/templates/upload_split.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-import argparse,sys,os, subprocess
+import argparse, sys, os, subprocess, time
 from subprocess import Popen, PIPE, STDOUT
 from io import BytesIO
 
@@ -7,7 +7,64 @@ ROOTURL="{{ rooturl }}"
 SHARE="{{ name }}"
 TOKEN="{{ token }}"
 
+class ETA():
+    def __init__(self,total):
+        self.total = total
+        self.memory = 360 # seconds
+        self.counts = []
+        self.timestamps = []
+        self.started = time.time()
+
+    def update(self,count):
+
+        self.counts.append(count)
+        self.timestamps.append(time.time())
+
+        newest = self.timestamps[-1]
+        delete_us = []
+        for i in range(len(self.timestamps)):
+            if newest - self.timestamps[i] > self.memory:
+                delete_us.append(i)
+        for delete in delete_us:
+            del self.counts[delete]
+            del self.timestamps[delete]
+
+    def get_seconds(self):
+        if len(self.timestamps)<2:
+            return None
+        diff_stamp = self.timestamps[-1] - self.timestamps[0]
+        diff_count = self.counts[-1] - self.counts[0]
+        eps = diff_count / diff_stamp
+
+        time_left = (self.total - self.counts[-1]) / eps
+        return time_left
+
+    def get_eta(self):
+        return time.strftime(
+            '%H:%M:%S',
+            time.gmtime(
+                self.get_seconds()
+            )
+        )
+
+    def get_finished(self):
+        return time.strftime(
+            '%H:%M:%S',
+            time.gmtime(
+                time.time() - self.started
+            )
+        )
+
+
 def split_upload(path, opts):
+    try:
+        size = int(subprocess.check_output(['du','-b',path]).split("\t")[0])
+    except KeyboardInterrupt:
+        size = 0
+    eta = ETA(size)
+    eta.update(0)
+    split_bytes = opts.split * 1024 * 1024
+    parts_estimate = -(-size // split_bytes) # clever ceil
     if os.path.isdir(path):
         tar = Popen(
             [
@@ -16,7 +73,11 @@ def split_upload(path, opts):
             stdout = PIPE
         )
         reader = tar.stdout
-        basename = os.path.basename(path.rstrip("/")) + ".tar"
+        basename = os.path.basename(
+            os.path.abspath(
+                path
+            )
+        ) + ".tar"
     elif os.path.isfile(path):
         reader = open(path, 'rb')
         basename = os.path.basename(path)
@@ -25,18 +86,19 @@ def split_upload(path, opts):
         return
 
     try:
-        chunk = reader.read(opts.split * 1024 * 1024)
+        chunk = reader.read(split_bytes)
         part = 0
+        eta_str = "ETA"
         while chunk != "":
             chunk_name = ".%s.part.%03d"%(
                 basename,
                 part
             )
-            print("%s part %d"%( basename, part ))
+            print("%s part %d/%d [%s]"%( basename, part + 1, parts_estimate, eta_str ))
             if not is_chunk_sent(chunk_name, opts):
                 p = Popen(
                     [
-                        'curl','-f',
+                        'curl','-s',
                         '-F','file=@-;filename=%s'%(chunk_name,),
                         '%supload/%s/%s'%(opts.rooturl, opts.share, opts.token)
                     ],
@@ -45,13 +107,15 @@ def split_upload(path, opts):
                     stderr=PIPE
                 )
                 stdout_data, stderr_data = p.communicate(input=chunk)
-                print(stdout_data)
+                if len(stderr_data) > 0:
-
+                    print(stderr_data)
-            chunk = reader.read(opts.split * 1024 * 1024)
+            chunk = reader.read(split_bytes)
             part += 1
+            eta.update(part * split_bytes)
+            eta_str = eta.get_eta()
     finally:
         reader.close()
-    join_chunks(basename,opts)
+    join_chunks(basename,part,opts)
     return
 
 
@@ -68,11 +132,12 @@ def is_chunk_sent(name, opts):
     return stdout_data == str(opts.split * 1024 * 1024)
 
 
-def join_chunks(name,opts):
+def join_chunks(name,parts,opts):
     p = Popen(
         [
             'curl',
             '-F','filename=%s'%(name,),
+            '-F','parts=%d'%(parts,),
             '%supload_join/%s/%s'%(opts.rooturl, opts.share, opts.token)
         ],
         stdout=PIPE,
@@ -84,7 +149,7 @@ def join_chunks(name,opts):
 
 def parse_options():
     parser = argparse.ArgumentParser(description='Flees uploader')
-    parser.add_argument('-s', action="store", type=int, dest="split", default = 64,
+    parser.add_argument('-s', action="store", type=int, dest="split", default = 32,
         help  = "Split size in megabytes [%(default)s]"
     )
     parser.add_argument('--rooturl', action="store", dest="rooturl",