175 lines
4.8 KiB
Python
175 lines
4.8 KiB
Python
#!/usr/bin/env python
|
|
import argparse, sys, os, subprocess, time
|
|
from subprocess import Popen, PIPE, STDOUT
|
|
from io import BytesIO
|
|
|
|
ROOTURL="{{ rooturl }}"
|
|
SHARE="{{ name }}"
|
|
TOKEN="{{ token }}"
|
|
|
|
class ETA():
|
|
def __init__(self,total):
|
|
self.total = total
|
|
self.memory = 360 # seconds
|
|
self.counts = []
|
|
self.timestamps = []
|
|
self.started = time.time()
|
|
|
|
def update(self,count):
|
|
|
|
self.counts.append(count)
|
|
self.timestamps.append(time.time())
|
|
|
|
newest = self.timestamps[-1]
|
|
delete_us = []
|
|
for i in range(len(self.timestamps)):
|
|
if newest - self.timestamps[i] > self.memory:
|
|
delete_us.append(i)
|
|
for delete in delete_us:
|
|
del self.counts[delete]
|
|
del self.timestamps[delete]
|
|
|
|
def get_seconds(self):
|
|
if len(self.timestamps)<2:
|
|
return None
|
|
diff_stamp = self.timestamps[-1] - self.timestamps[0]
|
|
diff_count = self.counts[-1] - self.counts[0]
|
|
eps = diff_count / diff_stamp
|
|
|
|
time_left = (self.total - self.counts[-1]) / eps
|
|
return time_left
|
|
|
|
def get_eta(self):
|
|
return time.strftime(
|
|
'%H:%M:%S',
|
|
time.gmtime(
|
|
self.get_seconds()
|
|
)
|
|
)
|
|
|
|
def get_finished(self):
|
|
return time.strftime(
|
|
'%H:%M:%S',
|
|
time.gmtime(
|
|
time.time() - self.started
|
|
)
|
|
)
|
|
|
|
|
|
def split_upload(path, opts):
|
|
try:
|
|
size = int(subprocess.check_output(['du','-b',path]).split("\t")[0])
|
|
except KeyboardInterrupt:
|
|
size = 0
|
|
eta = ETA(size)
|
|
eta.update(0)
|
|
split_bytes = opts.split * 1024 * 1024
|
|
parts_estimate = -(-size // split_bytes) # clever ceil
|
|
if os.path.isdir(path):
|
|
tar = Popen(
|
|
[
|
|
'tar','c',path
|
|
],
|
|
stdout = PIPE
|
|
)
|
|
reader = tar.stdout
|
|
basename = os.path.basename(
|
|
os.path.abspath(
|
|
path
|
|
)
|
|
) + ".tar"
|
|
elif os.path.isfile(path):
|
|
reader = open(path, 'rb')
|
|
basename = os.path.basename(path)
|
|
else:
|
|
print("Path %s doesnt exist"%( path, ))
|
|
return
|
|
|
|
try:
|
|
chunk = reader.read(split_bytes)
|
|
part = 0
|
|
eta_str = "ETA"
|
|
while chunk != "":
|
|
chunk_name = ".%s.part.%03d"%(
|
|
basename,
|
|
part
|
|
)
|
|
print("%s part %d/%d [%s]"%( basename, part + 1, parts_estimate, eta_str ))
|
|
if not is_chunk_sent(chunk_name, opts):
|
|
p = Popen(
|
|
[
|
|
'curl','-s',
|
|
'-F','file=@-;filename=%s'%(chunk_name,),
|
|
'%supload/%s/%s'%(opts.rooturl, opts.share, opts.token)
|
|
],
|
|
stdout=PIPE,
|
|
stdin=PIPE,
|
|
stderr=PIPE
|
|
)
|
|
stdout_data, stderr_data = p.communicate(input=chunk)
|
|
if len(stderr_data) > 0:
|
|
print(stderr_data)
|
|
chunk = reader.read(split_bytes)
|
|
part += 1
|
|
eta.update(part * split_bytes)
|
|
eta_str = eta.get_eta()
|
|
finally:
|
|
reader.close()
|
|
join_chunks(basename,part,opts)
|
|
return
|
|
|
|
|
|
def is_chunk_sent(name, opts):
|
|
p = Popen(
|
|
[
|
|
'curl','-s',
|
|
'%sfile/size/%s/%s/%s'%(opts.rooturl, opts.share, opts.token, name)
|
|
],
|
|
stdout=PIPE,
|
|
stderr=PIPE
|
|
)
|
|
stdout_data, stderr_data = p.communicate()
|
|
return stdout_data == str(opts.split * 1024 * 1024)
|
|
|
|
|
|
def join_chunks(name,parts,opts):
|
|
p = Popen(
|
|
[
|
|
'curl',
|
|
'-F','filename=%s'%(name,),
|
|
'-F','parts=%d'%(parts,),
|
|
'%supload_join/%s/%s'%(opts.rooturl, opts.share, opts.token)
|
|
],
|
|
stdout=PIPE,
|
|
stderr=PIPE
|
|
)
|
|
stdout_data, stderr_data = p.communicate()
|
|
print(stdout_data)
|
|
|
|
|
|
def parse_options():
|
|
parser = argparse.ArgumentParser(description='Flees uploader')
|
|
parser.add_argument('-s', action="store", type=int, dest="split", default = 32,
|
|
help = "Split size in megabytes [%(default)s]"
|
|
)
|
|
parser.add_argument('--rooturl', action="store", dest="rooturl",
|
|
default = ROOTURL,
|
|
help = "Address of Flees server [%(default)s]"
|
|
)
|
|
parser.add_argument('--share', action="store", dest="share",
|
|
default = SHARE,
|
|
help = "Name of Flees share [%(default)s]"
|
|
)
|
|
parser.add_argument('--token', action="store", dest="token",
|
|
default = TOKEN,
|
|
help = "API token for the share [%(default)s]"
|
|
)
|
|
parser.add_argument('path', nargs='+')
|
|
return parser.parse_args()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
opts = parse_options()
|
|
for path in opts.path:
|
|
split_upload(path,opts)
|