split upload with moar features

code/templates/upload_split.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-import argparse,sys,os, subprocess
+import argparse, sys, os, subprocess, time
 from subprocess import Popen, PIPE, STDOUT
 from io import BytesIO
 
@@ -7,7 +7,64 @@ ROOTURL="{{ rooturl }}"
 SHARE="{{ name }}"
 TOKEN="{{ token }}"
 
+class ETA():
+    def __init__(self,total):
+        self.total = total
+        self.memory = 360 # seconds
+        self.counts = []
+        self.timestamps = []
+        self.started = time.time()
+
+    def update(self,count):
+
+        self.counts.append(count)
+        self.timestamps.append(time.time())
+
+        newest = self.timestamps[-1]
+        delete_us = []
+        for i in range(len(self.timestamps)):
+            if newest - self.timestamps[i] > self.memory:
+                delete_us.append(i)
+        for delete in delete_us:
+            del self.counts[delete]
+            del self.timestamps[delete]
+
+    def get_seconds(self):
+        if len(self.timestamps)<2:
+            return None
+        diff_stamp = self.timestamps[-1] - self.timestamps[0]
+        diff_count = self.counts[-1] - self.counts[0]
+        eps = diff_count / diff_stamp
+
+        time_left = (self.total - self.counts[-1]) / eps
+        return time_left
+
+    def get_eta(self):
+        return time.strftime(
+            '%H:%M:%S',
+            time.gmtime(
+                self.get_seconds()
+            )
+        )
+
+    def get_finished(self):
+        return time.strftime(
+            '%H:%M:%S',
+            time.gmtime(
+                time.time() - self.started
+            )
+        )
+
+
 def split_upload(path, opts):
+    try:
+        size = int(subprocess.check_output(['du','-b',path]).split("\t")[0])
+    except KeyboardInterrupt:
+        size = 0
+    eta = ETA(size)
+    eta.update(0)
+    split_bytes = opts.split * 1024 * 1024
+    parts_estimate = -(-size // split_bytes) # clever ceil
     if os.path.isdir(path):
         tar = Popen(
             [
@@ -16,7 +73,11 @@ def split_upload(path, opts):
             stdout = PIPE
         )
         reader = tar.stdout
-        basename = os.path.basename(path.rstrip("/")) + ".tar"
+        basename = os.path.basename(
+            os.path.abspath(
+                path
+            )
+        ) + ".tar"
     elif os.path.isfile(path):
         reader = open(path, 'rb')
         basename = os.path.basename(path)
@@ -25,18 +86,19 @@ def split_upload(path, opts):
         return
 
     try:
-        chunk = reader.read(opts.split * 1024 * 1024)
+        chunk = reader.read(split_bytes)
         part = 0
+        eta_str = "ETA"
         while chunk != "":
             chunk_name = ".%s.part.%03d"%(
                 basename,
                 part
             )
-            print("%s part %d"%( basename, part ))
+            print("%s part %d/%d [%s]"%( basename, part + 1, parts_estimate, eta_str ))
             if not is_chunk_sent(chunk_name, opts):
                 p = Popen(
                     [
-                        'curl','-f',
+                        'curl','-s',
                         '-F','file=@-;filename=%s'%(chunk_name,),
                         '%supload/%s/%s'%(opts.rooturl, opts.share, opts.token)
                     ],
@@ -45,13 +107,15 @@ def split_upload(path, opts):
                     stderr=PIPE
                 )
                 stdout_data, stderr_data = p.communicate(input=chunk)
-                print(stdout_data)
-
-            chunk = reader.read(opts.split * 1024 * 1024)
+                if len(stderr_data) > 0:
+                    print(stderr_data)
+            chunk = reader.read(split_bytes)
             part += 1
+            eta.update(part * split_bytes)
+            eta_str = eta.get_eta()
     finally:
         reader.close()
-    join_chunks(basename,opts)
+    join_chunks(basename,part,opts)
     return
 
 
@@ -68,11 +132,12 @@ def is_chunk_sent(name, opts):
     return stdout_data == str(opts.split * 1024 * 1024)
 
 
-def join_chunks(name,opts):
+def join_chunks(name,parts,opts):
     p = Popen(
         [
             'curl',
             '-F','filename=%s'%(name,),
+            '-F','parts=%d'%(parts,),
             '%supload_join/%s/%s'%(opts.rooturl, opts.share, opts.token)
         ],
         stdout=PIPE,
@@ -84,7 +149,7 @@ def join_chunks(name,opts):
 
 def parse_options():
     parser = argparse.ArgumentParser(description='Flees uploader')
-    parser.add_argument('-s', action="store", type=int, dest="split", default = 64,
+    parser.add_argument('-s', action="store", type=int, dest="split", default = 32,
         help  = "Split size in megabytes [%(default)s]"
     )
     parser.add_argument('--rooturl', action="store", dest="rooturl",