too much stuff. uploader script from template. dropdown menu for tools

code/app.py

@@ -4,7 +4,8 @@ import os,sys,time,stat
 import json
 from datetime import datetime
 from flask import Flask, render_template, jsonify, current_app, Response, \
-    redirect, url_for, request, g, session, send_file, send_from_directory
+    redirect, url_for, request, g, session, send_file, send_from_directory, \
+    abort
 from werkzeug.utils import secure_filename
 import zipfile
 from multiprocessing import Process
@@ -98,7 +99,7 @@ def upload(name = None, token = None):
         if file:
             filename = os.path.join(
                 share['path'],
-                secure_filename(
+                secure_filename_hidden(
                     file.filename
                 )
             )
@@ -133,10 +134,10 @@ def upload_join_splitted(name, token):
         if not 'filename' in request.form:
             return "No filename given", 400
         parts = []
-        for part in range(100):
+        for part in range(500):
             filename = os.path.join(
                 share['path'],
-                "%s.part.%03d"%(
+                ".%s.part.%03d"%(
                     request.form['filename'],
                     part
                 )
@@ -145,7 +146,7 @@ def upload_join_splitted(name, token):
                 parts.append(filename)
                 part_existed = part
         if len(parts) == 0:
-            return "Invalid partial filename\n", 400
+            return "Invalid partial filename %s -> %s\n"%( request.form['filename'], filename), 400
         if not len(parts) == part_existed + 1:
             return "Parts missing\n", 400
         target_name = os.path.join(
@@ -163,6 +164,41 @@ def upload_join_splitted(name, token):
         return "Joining started\n", 200
 
 
+@app.route('/upload/url', methods=['POST'])
+def upload_url():
+    if request.method == 'POST':
+        name = request.form['name']
+        url = request.form['url']
+        if not is_valid_url(url):
+            return "URL not valid", 400
+        (ok,share) = get_share(name)
+        if not ok:
+            return share
+        if not get_or_none('upload', share) == True:
+            return "Upload not allowed\n",400
+        filename = os.path.join(
+            share['path'],
+            secure_filename(
+                os.path.basename(url)
+            )
+        )
+        if os.path.exists(filename):
+            file_versionize(filename)
+        download_url(url, filename)
+        set_rights(filename)
+        notify({
+            "recipient": get_or_none('recipient', share),
+            "share": name,
+            "filename": filename,
+            "operation": "upload"
+        })
+        if 'from_gui' in request.form:
+            if request.form['from_gui'] == "true":
+                return redirect(url_for('list_view',name=name))
+        return "File uploaded\n", 200
+
+
+
 @app.route('/send/<name>', methods=['GET'])
 def send(name):
     (ok,share) = get_share(name)
@@ -171,7 +207,22 @@ def send(name):
     return render_template('send.html',name=name)
 
 
-@app.route('/files/<name>/<token>', methods=['GET'])
+@app.route('/file/size/<name>/<token>/<path:filename>', methods=['GET'])
+def file_size(name, token, filename):
+    (ok,share) = get_share(name, token = token)
+    if not ok:
+        return share
+    full_path = os.path.join(
+        share['path'],
+        secure_filename_hidden(filename)
+    )
+    if not os.path.exists(full_path):
+        return "-1", 200
+    size = os.stat(full_path).st_size
+    return  str(size), 200
+
+
+@app.route('/file/list/<name>/<token>', methods=['GET'])
 def list_files(name, token):
     (ok,share) = get_share(name, token = token)
     if not ok:
@@ -443,60 +494,99 @@ def script_upload_split(name = None, token = None):
         return share
     if not get_or_none('upload', share) == True:
         return "Upload not allowed",400
-    return """#!/bin/bash
+    return render_template(
-test -n "$1" || {
+        "upload_split.py",
-  echo "Usage: [-s SplitMegabytes] file/folder [files/folders]"
+        name = name,
-  exit 1
+        token = token,
-}
+        rooturl = request.url_root
-MAXBYTES=$(( 512 * 1024 * 1024 ))
-for (( i=1; i<=$#; i++ )); do
-  j=$(( $i + 1 ))
-  [[ ${!i} = "-s" ]] && {
-    MAXBYTES=$(( ${!j} * 1024 * 1024 ))
-    shift 2
-  }
-done
-CAT=$( which cat )
-which pv &> /dev/null && CAT=$( which pv )
-ROOTURL="%s"
-SHARE="%s"
-TOKEN="%s"
-
-send_file() {
-  $CAT "$file_name" | split -d -a 3 -b $MAXBYTES \
-    --filter="curl -f -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \
-    - "$base_name.part." && \
-    curl -F "filename=$base_name" ${ROOTURL}upload_join/${SHARE}/${TOKEN}
-}
-send_folder() {
-  which pv &> /dev/null && printf -v dusize -- "--size %%dk" $( du -s -k "$file_name" | cut -f1 )
-  tar c "$file_name" | $CAT $dusize - | split -d -a 3 -b $MAXBYTES \
-    --filter="curl -f -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \
-    - "$base_name.tar.part." && \
-    curl -F "filename=$base_name.tar" ${ROOTURL}upload_join/${SHARE}/${TOKEN}
-}
-
-echo "Splitting to $(( $MAXBYTES / 1024 / 1024 )) Mb chunks"
-for file_name in "$@"; do
-  base_name=$( basename "$file_name" )
-  test -f "$file_name" && {
-      printf "Sending file: %%s\n" "$file_name"
-      send_file
-      continue
-  }
-  test -d "$file_name" && {
-      printf "Sending folder: %%s\n" "$file_name"
-      send_folder
-      continue
-  }
-done
-    """%(
-        request.url_root,
-        name,
-        token
     )
 
 
+
+#~ mmmmm                                     #      ""#
+#~ #   "#  mmm    mmm   m   m  mmmmm   mmm   #mmm     #     mmm
+#~ #mmmm" #"  #  #   "  #   #  # # #  "   #  #" "#    #    #"  #
+#~ #   "m #""""   """m  #   #  # # #  m"""#  #   #    #    #""""
+#~ #    " "#mm"  "mmm"  "mm"#  # # #  "mm"#  ##m#"    "mm  "#mm"
+
+#~ @app.route("/resumable/send")
+#~ def resumable_example():
+    #~ return render_template("resumable_upload.html")
+
+#~ # resumable.js uses a GET request to check if it uploaded the file already.
+#~ # NOTE: your validation here needs to match whatever you do in the POST (otherwise it will NEVER find the files)
+#~ @app.route("/resumable/get", methods=['GET'])
+#~ def resumable():
+    #~ resumableIdentfier = request.args.get('resumableIdentifier', type=str)
+    #~ resumableFilename = request.args.get('resumableFilename', type=str)
+    #~ resumableChunkNumber = request.args.get('resumableChunkNumber', type=int)
+
+    #~ if not resumableIdentfier or not resumableFilename or not resumableChunkNumber:
+        #~ # Parameters are missing or invalid
+        #~ abort(500, 'Parameter error')
+
+    #~ # chunk folder path based on the parameters
+    #~ temp_dir = os.path.join(temp_base, resumableIdentfier)
+
+    #~ # chunk path based on the parameters
+    #~ chunk_file = os.path.join(temp_dir, get_chunk_name(resumableFilename, resumableChunkNumber))
+    #~ app.logger.debug('Getting chunk: %s', chunk_file)
+
+    #~ if os.path.isfile(chunk_file):
+        #~ # Let resumable.js know this chunk already exists
+        #~ return 'OK'
+    #~ else:
+        #~ # Let resumable.js know this chunk does not exists and needs to be uploaded
+        #~ abort(404, 'Not found')
+
+
+#~ # if it didn't already upload, resumable.js sends the file here
+#~ @app.route("/resumable/post", methods=['POST'])
+#~ def resumable_post():
+    #~ resumableTotalChunks = request.form.get('resumableTotalChunks', type=int)
+    #~ resumableChunkNumber = request.form.get('resumableChunkNumber', default=1, type=int)
+    #~ resumableFilename = request.form.get('resumableFilename', default='error', type=str)
+    #~ resumableIdentfier = request.form.get('resumableIdentifier', default='error', type=str)
+
+    #~ # get the chunk data
+    #~ chunk_data = request.files['file']
+
+    #~ # make our temp directory
+    #~ temp_dir = os.path.join(temp_base, resumableIdentfier)
+    #~ if not os.path.isdir(temp_dir):
+        #~ os.makedirs(temp_dir, 0777)
+
+    #~ # save the chunk data
+    #~ chunk_name = get_chunk_name(resumableFilename, resumableChunkNumber)
+    #~ chunk_file = os.path.join(temp_dir, chunk_name)
+    #~ chunk_data.save(chunk_file)
+    #~ app.logger.debug('Saved chunk: %s', chunk_file)
+
+    #~ # check if the upload is complete
+    #~ chunk_paths = [os.path.join(temp_dir, get_chunk_name(resumableFilename, x)) for x in range(1, resumableTotalChunks+1)]
+    #~ upload_complete = all([os.path.exists(p) for p in chunk_paths])
+
+    #~ # combine all the chunks to create the final file
+    #~ if upload_complete:
+        #~ target_file_name = os.path.join(temp_base, resumableFilename)
+        #~ with open(target_file_name, "ab") as target_file:
+            #~ for p in chunk_paths:
+                #~ stored_chunk_file_name = p
+                #~ stored_chunk_file = open(stored_chunk_file_name, 'rb')
+                #~ target_file.write(stored_chunk_file.read())
+                #~ stored_chunk_file.close()
+                #~ os.unlink(stored_chunk_file_name)
+        #~ target_file.close()
+        #~ os.rmdir(temp_dir)
+        #~ app.logger.debug('File saved to: %s', target_file_name)
+
+    #~ return 'OK'
+
+
+#~ def get_chunk_name(uploaded_filename, chunk_number):
+    #~ return uploaded_filename + "_part_%03d" % chunk_number
+
+
 class uploadJoiner:
     def __init__(self, target_name, parts):
         self.target_name = target_name
@@ -623,6 +713,13 @@ def notify(msg):
         app.config['notifier'].notify(msg)
 
 
+def secure_filename_hidden(filename):
+    secure = secure_filename(filename)
+    if filename.startswith("."):
+        secure = "." + secure
+    return secure
+
+
 def set_rights(path):
     os.chown(path, app.config['UID'], app.config['GID'])
     st = os.stat(path)

code/static/css/styles.css

@@ -102,8 +102,23 @@ tr:nth-child(odd) {
   width: 200px;
 }
 #list_upload_button {
-  position: absolute;
+  float: right;
-  right: 8px;
+}
+#list_url_upload_text {
+  width: 95%;
+  float: right;
+}
+#list_url_upload_button {
+  float: right;
+}
+
+#list_info_toggle {
+  margin-top: 24px;
+  cursor: pointer;
+  text-align: right;
+}
+#list_info {
+  display: none;
 }
 
 #list_table {

code/static/js/scripts.js

@@ -1,4 +1,12 @@
 
+function clear_text(id,defaultValue) {
+    var element = document.getElementById(id);
+    if ( element.value == defaultValue ) {
+        element.value = '';
+    }
+}
+
+
 function index_form_enter(event) {
 	if (event.which || event.keyCode) {
 		if ((event.which == 13) || (event.keyCode == 13)) {
@@ -12,6 +20,13 @@ function index_form_submit() {
         document.getElementById("index_form_name").value;
 }
 
+
+function infoToggle() {
+    var info = document.getElementById("list_info");
+    info.style.display = info.style.display === 'block' ? 'none' : 'block';
+}
+
+
 function UploadFile(file,file_no,files_total) {
     if (uploadTurn != file_no) {
         // Wait for our turn to upload. check every 0.5s

code/templates/list.html

@@ -9,7 +9,7 @@
             <input id="list_upload_select" type=file name=file
                    onchange="FileSelectHandler(event)" multiple="multiple"><br>
             <input id="list_upload_button" type=submit value=Upload>
-            <div id="progress"></div>
+            <div id="progress" class="clear"></div>
         </form>
     </div>
   {% else %}
@@ -18,8 +18,19 @@
             <input type=submit value=Upload disabled>
     </div>
   {% endif %}
+    <div id=list_info_toggle onclick="infoToggle()">Share tools</div>
     <div id=list_info>
-   Share:
+      <div id=list_url_upload>
+        <form id="url_upload_form" action={{ url_for('upload_url') }} method=post>
+            <input id="list_upload_name" type=hidden name=name value="{{ name|safe }}" />
+            <input type=hidden name=from_gui value="true" />
+            <input id="list_url_upload_text" type=text name=url
+                   value="https://..." onclick="clear_text(this.id,'https://...')"><br>
+            <input id="list_url_upload_button" type=submit value="Upload URL">
+            <div class="clear"></div>
+        </form>
+      </div>
+
     <ul>
   {% if public %}
     <li>is <a href="{{ url_for('index') }}" title="Share is publicly visible in the index">public</a>

code/templates/upload_split.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+import argparse,sys,os, subprocess
+from subprocess import Popen, PIPE, STDOUT
+from io import BytesIO
+
+ROOTURL="{{ rooturl }}"
+SHARE="{{ name }}"
+TOKEN="{{ token }}"
+
+def split_upload(path, opts):
+    if os.path.isdir(path):
+        tar = Popen(
+            [
+                'tar','c',path
+            ],
+            stdout = PIPE
+        )
+        reader = tar.stdout
+        basename = os.path.basename(path.rstrip("/")) + ".tar"
+    elif os.path.isfile(path):
+        reader = open(path, 'rb')
+        basename = os.path.basename(path)
+    else:
+        print("Path %s doesnt exist"%( path, ))
+        return
+
+    try:
+        chunk = reader.read(opts.split * 1024 * 1024)
+        part = 0
+        while chunk != "":
+            chunk_name = ".%s.part.%03d"%(
+                basename,
+                part
+            )
+            print("%s part %d"%( basename, part ))
+            if not is_chunk_sent(chunk_name, opts):
+                p = Popen(
+                    [
+                        'curl','-f',
+                        '-F','file=@-;filename=%s'%(chunk_name,),
+                        '%supload/%s/%s'%(opts.rooturl, opts.share, opts.token)
+                    ],
+                    stdout=PIPE,
+                    stdin=PIPE,
+                    stderr=PIPE
+                )
+                stdout_data, stderr_data = p.communicate(input=chunk)
+                print(stdout_data)
+
+            chunk = reader.read(opts.split * 1024 * 1024)
+            part += 1
+    finally:
+        reader.close()
+    join_chunks(basename,opts)
+    return
+
+
+def is_chunk_sent(name, opts):
+    p = Popen(
+        [
+            'curl','-s',
+            '%sfile/size/%s/%s/%s'%(opts.rooturl, opts.share, opts.token, name)
+        ],
+        stdout=PIPE,
+        stderr=PIPE
+    )
+    stdout_data, stderr_data = p.communicate()
+    return stdout_data == str(opts.split * 1024 * 1024)
+
+
+def join_chunks(name,opts):
+    p = Popen(
+        [
+            'curl',
+            '-F','filename=%s'%(name,),
+            '%supload_join/%s/%s'%(opts.rooturl, opts.share, opts.token)
+        ],
+        stdout=PIPE,
+        stderr=PIPE
+    )
+    stdout_data, stderr_data = p.communicate()
+    print(stdout_data)
+
+
+def parse_options():
+    parser = argparse.ArgumentParser(description='Flees uploader')
+    parser.add_argument('-s', action="store", type=int, dest="split", default = 64,
+        help  = "Split size in megabytes [%(default)s]"
+    )
+    parser.add_argument('--rooturl', action="store", dest="rooturl",
+        default = ROOTURL,
+        help  = "Address of Flees server [%(default)s]"
+    )
+    parser.add_argument('--share', action="store", dest="share",
+        default = SHARE,
+        help  = "Name of Flees share [%(default)s]"
+    )
+    parser.add_argument('--token', action="store", dest="token",
+        default = TOKEN,
+        help  = "API token for the share [%(default)s]"
+    )
+    parser.add_argument('path', nargs='+')
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    opts = parse_options()
+    for path in opts.path:
+        split_upload(path,opts)

code/utils/utils.py

@@ -3,8 +3,18 @@ from datetime import datetime
 from flask import current_app as app
 try:
     from urllib.request import pathname2url
+    from urllib.request import URLopener
+    from urllib.request import urlparse
 except ImportError:
     from urllib import pathname2url
+    from urllib import URLopener
+    from urlparse import urlparse
+
+
+def download_url(url,filename):
+    downloader = URLopener()
+    downloader.retrieve(url, filename)
+    return
 
 def file_date_human(num):
     return datetime.fromtimestamp(
@@ -61,6 +71,14 @@ def is_path_safe(path):
     return True
 
 
+def is_valid_url(url, qualifying = None):
+    min_attributes = ('scheme', 'netloc')
+    qualifying = min_attributes if qualifying is None else qualifying
+    token = urlparse(url)
+    return all([getattr(token, qualifying_attr)
+                for qualifying_attr in qualifying])
+
+
 def iter_folder_files(path, recursive = True):
     if recursive:
         for dirpath, dirnames, filenames in os.walk(path, topdown = False):