#!/usr/bin/env python import argparse, sys, os, subprocess, time from subprocess import Popen, PIPE, STDOUT from io import BytesIO ROOTURL="{{ rooturl }}" SHARE="{{ name }}" TOKEN="{{ token }}" class ETA(): def __init__(self,total): self.total = total self.memory_length = 360 # seconds self.timestamps = [] # (time, count) self.started = time.time() def update(self,count): self.timestamps.append((time.time(), count)) newest = self.timestamps[-1][0] self.timestamps = [x for x in self.timestamps if newest - x[0] < self.memory_length] def get_seconds(self): if len(self.timestamps)<2: return None diff_stamp = self.timestamps[-1][0] - self.timestamps[0][0] diff_count = self.timestamps[-1][1] - self.timestamps[0][1] eps = diff_count / diff_stamp time_left = (self.total - self.timestamps[-1][1]) / eps return time_left def get_eta(self): return self.format( self.get_seconds() ) def get_finished(self): return self.format( time.time() - self.started ) def format(self, seconds): return time.strftime( '%H:%M:%S', time.gmtime( seconds ) ) def split_upload(path, opts): try: size = int(subprocess.check_output(['du','-b',path]).split("\t")[0]) except KeyboardInterrupt: size = 0 eta = ETA(size) eta.update(0) split_bytes = opts.split * 1024 * 1024 parts_estimate = -(-size // split_bytes) # clever ceil if os.path.isdir(path): tar = Popen( [ 'tar','c',path ], stdout = PIPE ) reader = tar.stdout basename = os.path.basename( os.path.abspath( path ) ) + ".tar" elif os.path.isfile(path): reader = open(path, 'rb') basename = os.path.basename(path) else: print("Path %s doesnt exist"%( path, )) return try: chunk = reader.read(split_bytes) part = 0 eta_str = "ETA" while chunk != "": chunk_name = ".%s.part.%03d"%( basename, part ) print("%s part %d/%d [%s]"%( basename, part + 1, parts_estimate, eta_str )) if not is_chunk_sent(chunk_name, opts): p = Popen( [ 'curl','-s', '-F','file=@-;filename=%s'%(chunk_name,), '%supload/%s/%s'%(opts.rooturl, opts.share, opts.token) ], stdout=PIPE, stdin=PIPE, stderr=PIPE ) stdout_data, stderr_data = p.communicate(input=chunk) if len(stderr_data) > 0: print(stderr_data) chunk = reader.read(split_bytes) part += 1 eta.update(part * split_bytes) eta_str = eta.get_eta() finally: reader.close() join_chunks(basename,part,opts) return def is_chunk_sent(name, opts): p = Popen( [ 'curl','-s', '%sfile/size/%s/%s/%s'%(opts.rooturl, opts.share, opts.token, name) ], stdout=PIPE, stderr=PIPE ) stdout_data, stderr_data = p.communicate() return stdout_data == str(opts.split * 1024 * 1024) def join_chunks(name,parts,opts): p = Popen( [ 'curl', '-F','filename=%s'%(name,), '-F','parts=%d'%(parts,), '%supload_join/%s/%s'%(opts.rooturl, opts.share, opts.token) ], stdout=PIPE, stderr=PIPE ) stdout_data, stderr_data = p.communicate() print(stdout_data) def parse_options(): parser = argparse.ArgumentParser(description='Flees uploader') parser.add_argument('-s', action="store", type=int, dest="split", default = 32, help = "Split size in megabytes [%(default)s]" ) parser.add_argument('--rooturl', action="store", dest="rooturl", default = ROOTURL, help = "Address of Flees server [%(default)s]" ) parser.add_argument('--share', action="store", dest="share", default = SHARE, help = "Name of Flees share [%(default)s]" ) parser.add_argument('--token', action="store", dest="token", default = TOKEN, help = "API token for the share [%(default)s]" ) parser.add_argument('path', nargs='+') return parser.parse_args() if __name__ == "__main__": opts = parse_options() for path in opts.path: split_upload(path,opts)