From 86c855285c23b1cc80d1b07c077a1c8d5c8aed56 Mon Sep 17 00:00:00 2001 From: Ville Rantanen Date: Thu, 15 Mar 2018 12:14:32 +0200 Subject: [PATCH] too much stuff. uploader script from template. dropdown menu for tools --- code/app.py | 211 ++++++++++++++++++++++++--------- code/static/css/styles.css | 19 ++- code/static/js/scripts.js | 15 +++ code/templates/list.html | 43 ++++--- code/templates/upload_split.py | 109 +++++++++++++++++ code/utils/utils.py | 18 +++ 6 files changed, 340 insertions(+), 75 deletions(-) create mode 100644 code/templates/upload_split.py diff --git a/code/app.py b/code/app.py index 9f9d5ee..8efc49e 100644 --- a/code/app.py +++ b/code/app.py @@ -4,7 +4,8 @@ import os,sys,time,stat import json from datetime import datetime from flask import Flask, render_template, jsonify, current_app, Response, \ - redirect, url_for, request, g, session, send_file, send_from_directory + redirect, url_for, request, g, session, send_file, send_from_directory, \ + abort from werkzeug.utils import secure_filename import zipfile from multiprocessing import Process @@ -98,7 +99,7 @@ def upload(name = None, token = None): if file: filename = os.path.join( share['path'], - secure_filename( + secure_filename_hidden( file.filename ) ) @@ -133,10 +134,10 @@ def upload_join_splitted(name, token): if not 'filename' in request.form: return "No filename given", 400 parts = [] - for part in range(100): + for part in range(500): filename = os.path.join( share['path'], - "%s.part.%03d"%( + ".%s.part.%03d"%( request.form['filename'], part ) @@ -145,7 +146,7 @@ def upload_join_splitted(name, token): parts.append(filename) part_existed = part if len(parts) == 0: - return "Invalid partial filename\n", 400 + return "Invalid partial filename %s -> %s\n"%( request.form['filename'], filename), 400 if not len(parts) == part_existed + 1: return "Parts missing\n", 400 target_name = os.path.join( @@ -163,6 +164,41 @@ def upload_join_splitted(name, token): return "Joining started\n", 200 +@app.route('/upload/url', methods=['POST']) +def upload_url(): + if request.method == 'POST': + name = request.form['name'] + url = request.form['url'] + if not is_valid_url(url): + return "URL not valid", 400 + (ok,share) = get_share(name) + if not ok: + return share + if not get_or_none('upload', share) == True: + return "Upload not allowed\n",400 + filename = os.path.join( + share['path'], + secure_filename( + os.path.basename(url) + ) + ) + if os.path.exists(filename): + file_versionize(filename) + download_url(url, filename) + set_rights(filename) + notify({ + "recipient": get_or_none('recipient', share), + "share": name, + "filename": filename, + "operation": "upload" + }) + if 'from_gui' in request.form: + if request.form['from_gui'] == "true": + return redirect(url_for('list_view',name=name)) + return "File uploaded\n", 200 + + + @app.route('/send/', methods=['GET']) def send(name): (ok,share) = get_share(name) @@ -171,7 +207,22 @@ def send(name): return render_template('send.html',name=name) -@app.route('/files//', methods=['GET']) +@app.route('/file/size///', methods=['GET']) +def file_size(name, token, filename): + (ok,share) = get_share(name, token = token) + if not ok: + return share + full_path = os.path.join( + share['path'], + secure_filename_hidden(filename) + ) + if not os.path.exists(full_path): + return "-1", 200 + size = os.stat(full_path).st_size + return str(size), 200 + + +@app.route('/file/list//', methods=['GET']) def list_files(name, token): (ok,share) = get_share(name, token = token) if not ok: @@ -443,60 +494,99 @@ def script_upload_split(name = None, token = None): return share if not get_or_none('upload', share) == True: return "Upload not allowed",400 - return """#!/bin/bash -test -n "$1" || { - echo "Usage: [-s SplitMegabytes] file/folder [files/folders]" - exit 1 -} -MAXBYTES=$(( 512 * 1024 * 1024 )) -for (( i=1; i<=$#; i++ )); do - j=$(( $i + 1 )) - [[ ${!i} = "-s" ]] && { - MAXBYTES=$(( ${!j} * 1024 * 1024 )) - shift 2 - } -done -CAT=$( which cat ) -which pv &> /dev/null && CAT=$( which pv ) -ROOTURL="%s" -SHARE="%s" -TOKEN="%s" - -send_file() { - $CAT "$file_name" | split -d -a 3 -b $MAXBYTES \ - --filter="curl -f -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \ - - "$base_name.part." && \ - curl -F "filename=$base_name" ${ROOTURL}upload_join/${SHARE}/${TOKEN} -} -send_folder() { - which pv &> /dev/null && printf -v dusize -- "--size %%dk" $( du -s -k "$file_name" | cut -f1 ) - tar c "$file_name" | $CAT $dusize - | split -d -a 3 -b $MAXBYTES \ - --filter="curl -f -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \ - - "$base_name.tar.part." && \ - curl -F "filename=$base_name.tar" ${ROOTURL}upload_join/${SHARE}/${TOKEN} -} - -echo "Splitting to $(( $MAXBYTES / 1024 / 1024 )) Mb chunks" -for file_name in "$@"; do - base_name=$( basename "$file_name" ) - test -f "$file_name" && { - printf "Sending file: %%s\n" "$file_name" - send_file - continue - } - test -d "$file_name" && { - printf "Sending folder: %%s\n" "$file_name" - send_folder - continue - } -done - """%( - request.url_root, - name, - token + return render_template( + "upload_split.py", + name = name, + token = token, + rooturl = request.url_root ) + +#~ mmmmm # ""# +#~ # "# mmm mmm m m mmmmm mmm #mmm # mmm +#~ #mmmm" #" # # " # # # # # " # #" "# # #" # +#~ # "m #"""" """m # # # # # m"""# # # # #"""" +#~ # " "#mm" "mmm" "mm"# # # # "mm"# ##m#" "mm "#mm" + +#~ @app.route("/resumable/send") +#~ def resumable_example(): + #~ return render_template("resumable_upload.html") + +#~ # resumable.js uses a GET request to check if it uploaded the file already. +#~ # NOTE: your validation here needs to match whatever you do in the POST (otherwise it will NEVER find the files) +#~ @app.route("/resumable/get", methods=['GET']) +#~ def resumable(): + #~ resumableIdentfier = request.args.get('resumableIdentifier', type=str) + #~ resumableFilename = request.args.get('resumableFilename', type=str) + #~ resumableChunkNumber = request.args.get('resumableChunkNumber', type=int) + + #~ if not resumableIdentfier or not resumableFilename or not resumableChunkNumber: + #~ # Parameters are missing or invalid + #~ abort(500, 'Parameter error') + + #~ # chunk folder path based on the parameters + #~ temp_dir = os.path.join(temp_base, resumableIdentfier) + + #~ # chunk path based on the parameters + #~ chunk_file = os.path.join(temp_dir, get_chunk_name(resumableFilename, resumableChunkNumber)) + #~ app.logger.debug('Getting chunk: %s', chunk_file) + + #~ if os.path.isfile(chunk_file): + #~ # Let resumable.js know this chunk already exists + #~ return 'OK' + #~ else: + #~ # Let resumable.js know this chunk does not exists and needs to be uploaded + #~ abort(404, 'Not found') + + +#~ # if it didn't already upload, resumable.js sends the file here +#~ @app.route("/resumable/post", methods=['POST']) +#~ def resumable_post(): + #~ resumableTotalChunks = request.form.get('resumableTotalChunks', type=int) + #~ resumableChunkNumber = request.form.get('resumableChunkNumber', default=1, type=int) + #~ resumableFilename = request.form.get('resumableFilename', default='error', type=str) + #~ resumableIdentfier = request.form.get('resumableIdentifier', default='error', type=str) + + #~ # get the chunk data + #~ chunk_data = request.files['file'] + + #~ # make our temp directory + #~ temp_dir = os.path.join(temp_base, resumableIdentfier) + #~ if not os.path.isdir(temp_dir): + #~ os.makedirs(temp_dir, 0777) + + #~ # save the chunk data + #~ chunk_name = get_chunk_name(resumableFilename, resumableChunkNumber) + #~ chunk_file = os.path.join(temp_dir, chunk_name) + #~ chunk_data.save(chunk_file) + #~ app.logger.debug('Saved chunk: %s', chunk_file) + + #~ # check if the upload is complete + #~ chunk_paths = [os.path.join(temp_dir, get_chunk_name(resumableFilename, x)) for x in range(1, resumableTotalChunks+1)] + #~ upload_complete = all([os.path.exists(p) for p in chunk_paths]) + + #~ # combine all the chunks to create the final file + #~ if upload_complete: + #~ target_file_name = os.path.join(temp_base, resumableFilename) + #~ with open(target_file_name, "ab") as target_file: + #~ for p in chunk_paths: + #~ stored_chunk_file_name = p + #~ stored_chunk_file = open(stored_chunk_file_name, 'rb') + #~ target_file.write(stored_chunk_file.read()) + #~ stored_chunk_file.close() + #~ os.unlink(stored_chunk_file_name) + #~ target_file.close() + #~ os.rmdir(temp_dir) + #~ app.logger.debug('File saved to: %s', target_file_name) + + #~ return 'OK' + + +#~ def get_chunk_name(uploaded_filename, chunk_number): + #~ return uploaded_filename + "_part_%03d" % chunk_number + + class uploadJoiner: def __init__(self, target_name, parts): self.target_name = target_name @@ -623,6 +713,13 @@ def notify(msg): app.config['notifier'].notify(msg) +def secure_filename_hidden(filename): + secure = secure_filename(filename) + if filename.startswith("."): + secure = "." + secure + return secure + + def set_rights(path): os.chown(path, app.config['UID'], app.config['GID']) st = os.stat(path) diff --git a/code/static/css/styles.css b/code/static/css/styles.css index 667cff9..a7eaf49 100644 --- a/code/static/css/styles.css +++ b/code/static/css/styles.css @@ -102,8 +102,23 @@ tr:nth-child(odd) { width: 200px; } #list_upload_button { - position: absolute; - right: 8px; + float: right; +} +#list_url_upload_text { + width: 95%; + float: right; +} +#list_url_upload_button { + float: right; +} + +#list_info_toggle { + margin-top: 24px; + cursor: pointer; + text-align: right; +} +#list_info { + display: none; } #list_table { diff --git a/code/static/js/scripts.js b/code/static/js/scripts.js index 843f07a..056ab25 100644 --- a/code/static/js/scripts.js +++ b/code/static/js/scripts.js @@ -1,4 +1,12 @@ +function clear_text(id,defaultValue) { + var element = document.getElementById(id); + if ( element.value == defaultValue ) { + element.value = ''; + } +} + + function index_form_enter(event) { if (event.which || event.keyCode) { if ((event.which == 13) || (event.keyCode == 13)) { @@ -12,6 +20,13 @@ function index_form_submit() { document.getElementById("index_form_name").value; } + +function infoToggle() { + var info = document.getElementById("list_info"); + info.style.display = info.style.display === 'block' ? 'none' : 'block'; +} + + function UploadFile(file,file_no,files_total) { if (uploadTurn != file_no) { // Wait for our turn to upload. check every 0.5s diff --git a/code/templates/list.html b/code/templates/list.html index 6ce80d7..e041589 100644 --- a/code/templates/list.html +++ b/code/templates/list.html @@ -2,45 +2,56 @@ {% block body %}
{% if upload %} -
+

-
+
-
+
{% else %}
-
+
{% endif %} -
- Share: -
    +
    Share tools
    +
    +
    +
    + + +
    + +
    +
    +
    + +
      {% if public %} -
    • is public +
    • is public {% else %} -
    • is unlisted +
    • is unlisted {% endif %} {% if expire %} -
    • expires {{ expire }} +
    • expires {{ expire }} {% else %} -
    • never expires +
    • never expires {% endif %} {% if upload %} {% if overwrite %} -
    • uploads overwrite +
    • uploads overwrite {% else %} -
    • uploads versioned +
    • uploads versioned {% endif %} {% endif %} -
    • Download as zip -
    • Logout -
    +
  • Download as zip +
  • Logout +

{{ name }}

diff --git a/code/templates/upload_split.py b/code/templates/upload_split.py new file mode 100644 index 0000000..20ba2f7 --- /dev/null +++ b/code/templates/upload_split.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +import argparse,sys,os, subprocess +from subprocess import Popen, PIPE, STDOUT +from io import BytesIO + +ROOTURL="{{ rooturl }}" +SHARE="{{ name }}" +TOKEN="{{ token }}" + +def split_upload(path, opts): + if os.path.isdir(path): + tar = Popen( + [ + 'tar','c',path + ], + stdout = PIPE + ) + reader = tar.stdout + basename = os.path.basename(path.rstrip("/")) + ".tar" + elif os.path.isfile(path): + reader = open(path, 'rb') + basename = os.path.basename(path) + else: + print("Path %s doesnt exist"%( path, )) + return + + try: + chunk = reader.read(opts.split * 1024 * 1024) + part = 0 + while chunk != "": + chunk_name = ".%s.part.%03d"%( + basename, + part + ) + print("%s part %d"%( basename, part )) + if not is_chunk_sent(chunk_name, opts): + p = Popen( + [ + 'curl','-f', + '-F','file=@-;filename=%s'%(chunk_name,), + '%supload/%s/%s'%(opts.rooturl, opts.share, opts.token) + ], + stdout=PIPE, + stdin=PIPE, + stderr=PIPE + ) + stdout_data, stderr_data = p.communicate(input=chunk) + print(stdout_data) + + chunk = reader.read(opts.split * 1024 * 1024) + part += 1 + finally: + reader.close() + join_chunks(basename,opts) + return + + +def is_chunk_sent(name, opts): + p = Popen( + [ + 'curl','-s', + '%sfile/size/%s/%s/%s'%(opts.rooturl, opts.share, opts.token, name) + ], + stdout=PIPE, + stderr=PIPE + ) + stdout_data, stderr_data = p.communicate() + return stdout_data == str(opts.split * 1024 * 1024) + + +def join_chunks(name,opts): + p = Popen( + [ + 'curl', + '-F','filename=%s'%(name,), + '%supload_join/%s/%s'%(opts.rooturl, opts.share, opts.token) + ], + stdout=PIPE, + stderr=PIPE + ) + stdout_data, stderr_data = p.communicate() + print(stdout_data) + + +def parse_options(): + parser = argparse.ArgumentParser(description='Flees uploader') + parser.add_argument('-s', action="store", type=int, dest="split", default = 64, + help = "Split size in megabytes [%(default)s]" + ) + parser.add_argument('--rooturl', action="store", dest="rooturl", + default = ROOTURL, + help = "Address of Flees server [%(default)s]" + ) + parser.add_argument('--share', action="store", dest="share", + default = SHARE, + help = "Name of Flees share [%(default)s]" + ) + parser.add_argument('--token', action="store", dest="token", + default = TOKEN, + help = "API token for the share [%(default)s]" + ) + parser.add_argument('path', nargs='+') + return parser.parse_args() + + +if __name__ == "__main__": + opts = parse_options() + for path in opts.path: + split_upload(path,opts) diff --git a/code/utils/utils.py b/code/utils/utils.py index 70899d7..f1a028b 100644 --- a/code/utils/utils.py +++ b/code/utils/utils.py @@ -3,8 +3,18 @@ from datetime import datetime from flask import current_app as app try: from urllib.request import pathname2url + from urllib.request import URLopener + from urllib.request import urlparse except ImportError: from urllib import pathname2url + from urllib import URLopener + from urlparse import urlparse + + +def download_url(url,filename): + downloader = URLopener() + downloader.retrieve(url, filename) + return def file_date_human(num): return datetime.fromtimestamp( @@ -61,6 +71,14 @@ def is_path_safe(path): return True +def is_valid_url(url, qualifying = None): + min_attributes = ('scheme', 'netloc') + qualifying = min_attributes if qualifying is None else qualifying + token = urlparse(url) + return all([getattr(token, qualifying_attr) + for qualifying_attr in qualifying]) + + def iter_folder_files(path, recursive = True): if recursive: for dirpath, dirnames, filenames in os.walk(path, topdown = False):