diff --git a/code/app.py b/code/app.py index 52c6a0e..01a3b15 100644 --- a/code/app.py +++ b/code/app.py @@ -133,8 +133,14 @@ def upload_join_splitted(name, token): return "Upload not allowed",400 if not 'filename' in request.form: return "No filename given", 400 + if not 'parts' in request.form: + return "No parts count given", 400 + try: + no_parts = int(request.form['parts']) + except: + return "Parts not parseable", 400 parts = [] - for part in range(500): + for part in range(no_parts): filename = os.path.join( share['path'], ".%s.part.%03d"%( @@ -351,42 +357,11 @@ def script_upload(name = None, token = None): return share if not get_or_none('upload', share) == True: return "Upload not allowed",400 - return """#!/bin/bash -test -n "$1" || { - echo "Add files to upload as argument" - exit 1 -} -CAT=$( which cat ) -which pv &> /dev/null && CAT=$( which pv ) -ROOTURL="%s" -SHARE="%s" -TOKEN="%s" - -send_file() { - $CAT "$file_name" | curl -F "file=@-;filename=${base_name}" ${ROOTURL}upload/${SHARE}/${TOKEN} -} -send_folder() { - which pv &> /dev/null && printf -v dusize -- "--size %%dk" $( du -s -k "$file_name" | cut -f1 ) - tar cz "$file_name" | $CAT $dusize - | curl -F "file=@-;filename=${base_name}.tgz" ${ROOTURL}upload/${SHARE}/${TOKEN} -} - -for file_name in "$@"; do - base_name=$( basename "$file_name" ) - test -f "$file_name" && { - printf "Sending file: %%s\n" "$file_name" - send_file - continue - } - test -d "$file_name" && { - printf "Sending folder: %%s\n" "$file_name" - send_folder - continue - } -done - """%( - request.url_root, - name, - token + return render_template( + "upload.sh", + name = name, + token = token, + rooturl = request.url_root ) @@ -395,99 +370,40 @@ def script_download(name = None, token = None): (ok,share) = get_share(name, token = token) if not ok: return share - files = [] + commands = [] for file in iter_folder_files(share['path']): status = file_stat(share['path'], file) - files.append(status) - script = """#!/bin/bash -test "$1" = "-h" && { - echo "Add argument -f to overwrite files" - exit 0 -} -test "$1" = "-f" && FORCE=1 -which curl &> /dev/null || { - echo "curl required" - exit 1 -} -ROOTURL="%s" -SHARE="%s" -TOKEN="%s" -get_file() { - WRITE=0 - FILENAME="$1" - test "$FORCE" = "1" && WRITE=1 - test -f "${FILENAME}" || WRITE=1 - test "$WRITE" = "1" && { - echo Downloading ${FILENAME} - mkdir -p $( dirname "$FILENAME" ) - curl "${ROOTURL}download/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}" - } || { - echo Skipping ${FILENAME} - } -} -"""%( - request.url_root, - name, - token + commands.append('get_file "%s"'%( + status['url'], + )) + return render_template( + "download.sh", + name = name, + token = token, + rooturl = request.url_root, + commands = "\n".join(commands) ) - for file in files: - script += 'get_file "%s"\n'%( - file['url'], - ) - return script - @app.route('/script/direct//', methods=['GET']) def script_direct(name = None, token = None): (ok,share) = get_share(name, token = token) if not ok: return share - files = [] + commands = [] for file in iter_folder_files(share['path']): status = file_stat(share['path'], file) - status.update({ - 'token': get_direct_token(share, file) - }) - files.append(status) - script = """#!/bin/bash -test "$1" = "-h" && { - echo "Add argument -f to overwrite files" - exit 0 -} -test "$1" = "-f" && FORCE=1 -which curl &> /dev/null || { - echo "curl required" - exit 1 -} -ROOTURL="%s" -SHARE="%s" -get_file() { - WRITE=0 - FILENAME="$1" - TOKEN="$2" - test "$FORCE" = "1" && WRITE=1 - test -f "${FILENAME}" || WRITE=1 - test "$WRITE" = "1" && { - echo Downloading ${FILENAME} - mkdir -p $( dirname "$FILENAME" ) - curl "${ROOTURL}direct/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}" - } || { - echo Skipping ${FILENAME} - } -} -"""%( - request.url_root, - name, + commands.append('get_file "%s" "%s"'%( + status['url'], + get_direct_token(share, file) + )) + return render_template( + "download_direct.sh", + name = name, + rooturl = request.url_root, + commands = "\n".join(commands) ) - for file in files: - script += 'get_file "%s" "%s"\n'%( - file['url'], - file['token'], - ) - return script - @app.route('/script/upload_split//', methods=['GET']) def script_upload_split(name = None, token = None): diff --git a/code/templates/download.sh b/code/templates/download.sh new file mode 100644 index 0000000..e0259ea --- /dev/null +++ b/code/templates/download.sh @@ -0,0 +1,28 @@ +#!/bin/bash +test "$1" = "-h" && { + echo "Add argument -f to overwrite files" + exit 0 +} +test "$1" = "-f" && FORCE=1 +which curl &> /dev/null || { + echo "curl required" + exit 1 +} +ROOTURL="{{ rooturl }}" +SHARE="{{ name }}" +TOKEN="{{ token }}" + +get_file() { + WRITE=0 + FILENAME="$1" + test "$FORCE" = "1" && WRITE=1 + test -f "${FILENAME}" || WRITE=1 + test "$WRITE" = "1" && { + echo Downloading ${FILENAME} + mkdir -p $( dirname "$FILENAME" ) + curl "${ROOTURL}download/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}" + } || { + echo Skipping ${FILENAME} + } +} +{{ commands }} diff --git a/code/templates/download_direct.sh b/code/templates/download_direct.sh new file mode 100644 index 0000000..edf9eba --- /dev/null +++ b/code/templates/download_direct.sh @@ -0,0 +1,28 @@ +#!/bin/bash +test "$1" = "-h" && { + echo "Add argument -f to overwrite files" + exit 0 +} +test "$1" = "-f" && FORCE=1 +which curl &> /dev/null || { + echo "curl required" + exit 1 +} +ROOTURL="{{ rooturl }}" +SHARE="{{ name }}" + +get_file() { + WRITE=0 + FILENAME="$1" + TOKEN="$2" + test "$FORCE" = "1" && WRITE=1 + test -f "${FILENAME}" || WRITE=1 + test "$WRITE" = "1" && { + echo Downloading ${FILENAME} + mkdir -p $( dirname "$FILENAME" ) + curl "${ROOTURL}direct/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}" + } || { + echo Skipping ${FILENAME} + } +} +{{ commands }} diff --git a/code/templates/upload.sh b/code/templates/upload.sh new file mode 100644 index 0000000..25ae55e --- /dev/null +++ b/code/templates/upload.sh @@ -0,0 +1,32 @@ +#!/bin/bash +test -n "$1" || { + echo "Add files to upload as argument" + exit 1 +} +CAT=$( which cat ) +which pv &> /dev/null && CAT=$( which pv ) +ROOTURL="{{ rooturl }}" +SHARE="{{ name }}" +TOKEN="{{ token }}" + +send_file() { + $CAT "$file_name" | curl -F "file=@-;filename=${base_name}" ${ROOTURL}upload/${SHARE}/${TOKEN} +} +send_folder() { + which pv &> /dev/null && printf -v dusize -- "--size %dk" $( du -s -k "$file_name" | cut -f1 ) + tar cz "$file_name" | $CAT $dusize - | curl -F "file=@-;filename=${base_name}.tgz" ${ROOTURL}upload/${SHARE}/${TOKEN} +} + +for file_name in "$@"; do + base_name=$( basename "$file_name" ) + test -f "$file_name" && { + printf "Sending file: %s\n" "$file_name" + send_file + continue + } + test -d "$file_name" && { + printf "Sending folder: %s\n" "$file_name" + send_folder + continue + } +done diff --git a/code/templates/upload_split.py b/code/templates/upload_split.py index 20ba2f7..cd662a3 100644 --- a/code/templates/upload_split.py +++ b/code/templates/upload_split.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -import argparse,sys,os, subprocess +import argparse, sys, os, subprocess, time from subprocess import Popen, PIPE, STDOUT from io import BytesIO @@ -7,7 +7,64 @@ ROOTURL="{{ rooturl }}" SHARE="{{ name }}" TOKEN="{{ token }}" +class ETA(): + def __init__(self,total): + self.total = total + self.memory = 360 # seconds + self.counts = [] + self.timestamps = [] + self.started = time.time() + + def update(self,count): + + self.counts.append(count) + self.timestamps.append(time.time()) + + newest = self.timestamps[-1] + delete_us = [] + for i in range(len(self.timestamps)): + if newest - self.timestamps[i] > self.memory: + delete_us.append(i) + for delete in delete_us: + del self.counts[delete] + del self.timestamps[delete] + + def get_seconds(self): + if len(self.timestamps)<2: + return None + diff_stamp = self.timestamps[-1] - self.timestamps[0] + diff_count = self.counts[-1] - self.counts[0] + eps = diff_count / diff_stamp + + time_left = (self.total - self.counts[-1]) / eps + return time_left + + def get_eta(self): + return time.strftime( + '%H:%M:%S', + time.gmtime( + self.get_seconds() + ) + ) + + def get_finished(self): + return time.strftime( + '%H:%M:%S', + time.gmtime( + time.time() - self.started + ) + ) + + def split_upload(path, opts): + try: + size = int(subprocess.check_output(['du','-b',path]).split("\t")[0]) + except KeyboardInterrupt: + size = 0 + eta = ETA(size) + eta.update(0) + split_bytes = opts.split * 1024 * 1024 + parts_estimate = -(-size // split_bytes) # clever ceil if os.path.isdir(path): tar = Popen( [ @@ -16,7 +73,11 @@ def split_upload(path, opts): stdout = PIPE ) reader = tar.stdout - basename = os.path.basename(path.rstrip("/")) + ".tar" + basename = os.path.basename( + os.path.abspath( + path + ) + ) + ".tar" elif os.path.isfile(path): reader = open(path, 'rb') basename = os.path.basename(path) @@ -25,18 +86,19 @@ def split_upload(path, opts): return try: - chunk = reader.read(opts.split * 1024 * 1024) + chunk = reader.read(split_bytes) part = 0 + eta_str = "ETA" while chunk != "": chunk_name = ".%s.part.%03d"%( basename, part ) - print("%s part %d"%( basename, part )) + print("%s part %d/%d [%s]"%( basename, part + 1, parts_estimate, eta_str )) if not is_chunk_sent(chunk_name, opts): p = Popen( [ - 'curl','-f', + 'curl','-s', '-F','file=@-;filename=%s'%(chunk_name,), '%supload/%s/%s'%(opts.rooturl, opts.share, opts.token) ], @@ -45,13 +107,15 @@ def split_upload(path, opts): stderr=PIPE ) stdout_data, stderr_data = p.communicate(input=chunk) - print(stdout_data) - - chunk = reader.read(opts.split * 1024 * 1024) + if len(stderr_data) > 0: + print(stderr_data) + chunk = reader.read(split_bytes) part += 1 + eta.update(part * split_bytes) + eta_str = eta.get_eta() finally: reader.close() - join_chunks(basename,opts) + join_chunks(basename,part,opts) return @@ -68,11 +132,12 @@ def is_chunk_sent(name, opts): return stdout_data == str(opts.split * 1024 * 1024) -def join_chunks(name,opts): +def join_chunks(name,parts,opts): p = Popen( [ 'curl', '-F','filename=%s'%(name,), + '-F','parts=%d'%(parts,), '%supload_join/%s/%s'%(opts.rooturl, opts.share, opts.token) ], stdout=PIPE, @@ -84,7 +149,7 @@ def join_chunks(name,opts): def parse_options(): parser = argparse.ArgumentParser(description='Flees uploader') - parser.add_argument('-s', action="store", type=int, dest="split", default = 64, + parser.add_argument('-s', action="store", type=int, dest="split", default = 32, help = "Split size in megabytes [%(default)s]" ) parser.add_argument('--rooturl', action="store", dest="rooturl",