split utils, added manual tests

code/Dockerfile

@@ -11,7 +11,8 @@ RUN pip3 install -r /requirements.txt
 COPY static /code/static
 COPY templates /code/templates
 COPY utils/__init__.py /code/utils/__init__.py
-COPY utils/utils.py /code/utils/utils.py
+COPY utils/files.py /code/utils/files.py
+COPY utils/misc.py /code/utils/misc.py
 COPY utils/crypt.py /code/utils/crypt.py
 COPY revprox.py /code/revprox.py
 COPY notifier.py /code/notifier.py

code/app.py

@@ -10,8 +10,7 @@ from werkzeug.utils import secure_filename
 import zipfile
 from multiprocessing import Process
 from revprox import ReverseProxied
-from utils.utils import *
-from utils.crypt import *
+from utils import *
 
 
 __FLEES_VERSION__ = "20181124.0"
@@ -162,9 +161,10 @@ def upload_join_splitted(name, token):
             return "Invalid partial filename %s -> %s\n"%( request.form['filename'], filename), 400
         if not len(parts) == part_existed + 1:
             return "Parts missing\n", 400
+        secure_name = secure_filename(request.form['filename'])
         target_name = os.path.join(
             share['path'],
-            request.form['filename']
+            secure_name
         )
         if get_or_none('overwrite', share) == False:
             if os.path.exists(target_name):
@@ -174,7 +174,7 @@ def upload_join_splitted(name, token):
             begin = uploadJoiner(target_name, parts)
         except:
             return "Joining failed\n", 400
-        download_url = get_download_url(share, request.form['filename'], token)
+        download_url = get_download_url(share, secure_name, token)
         return "Joining started\n%s\n"%( download_url, ), 200
 
 
@@ -365,15 +365,7 @@ def file_direct(name, token, filename):
     allow_direct = get_or_none('direct_links', share) if get_or_none('pass_hash', share) else False
     if not allow_direct:
         return "-1", 403
-    token = get_direct_token(share, filename)
-    # url_for returns extra level of /../path if proxying with nginx
-    return  "/".join((
-        app.config['PUBLIC_URL'],
-        'direct',
-        name,
-        token,
-        path2url(filename)
-    )), 200
+    return get_download_url(share, filename, token), 200
 
 
 @app.route('/file/ls/<name>/<token>', methods=['GET'])
@@ -936,25 +928,6 @@ def zip_clean():
             os.remove(os.path.join(app.config['ZIP_FOLDER'],file))
 
 
-def get_download_url(share, file, token):
-    direct = get_or_none('direct_links', share, False)
-    if direct:
-        return  "/".join((
-            app.config['PUBLIC_URL'],
-            'direct',
-            share['name'],
-            get_direct_token(share, file),
-            path2url(file)
-        ))
-    else:
-        return "/".join((
-            app.config['PUBLIC_URL'],
-            'download',
-            share['name'],
-            token,
-            path2url(file)
-        ))
-
 
 if __name__ == "__main__":
     zip_clean()

code/flees-manager.py

@@ -3,8 +3,7 @@ import argparse,json,sys,os
 from shutil import copyfile
 from tabulate import tabulate
 from datetime import datetime
-from utils.utils import *
-from utils.crypt import *
+from utils import *
 
 
 def get_root_path(opts):
@@ -620,7 +619,7 @@ def print_rest_api_upload(config, share, token):
         token
     ))
     print("\nLink to upload multiple files to the share, splitting large files:")
-    print("\n# curl -s %s/script/upload_split/%s/%s | python - [-s split_size_in_Mb] file_to_upload.ext [second.file.ext]"%(
+    print("\n# python2 <( curl -s %s/script/upload_split/%s/%s ) [-s split_size_in_Mb] file_to_upload.ext [second.file.ext]"%(
         config['public_url'],
         share['name'],
         token

code/utils/__init__.py

@@ -1 +1,3 @@
-
+from utils.misc import *
+from utils.files import *
+from utils.crypt import *

code/utils/files.py

@@ -5,21 +5,14 @@ import requests
 import re
 import json
 import stat
+from .misc import *
+from .crypt import *
+
 try:
     import magic
 except ImportError:
     pass
-try:
-    from werkzeug.utils import secure_filename
-except ImportError:
-    pass
 
-try:
-    from urllib.request import pathname2url
-    from urllib.request import urlparse
-except ImportError:
-    from urllib import pathname2url
-    from urlparse import urlparse
 
 class Logger:
     def __init__(self, filename, uid = 0, gid = 0):
@@ -71,15 +64,6 @@ class Logger:
             fp.flush()
 
 
-def bool_short(var):
-    if type(var) == bool:
-        if var:
-            return "Y"
-        else:
-            return "N"
-    return var
-
-
 def download_url(url, filename):
     try:
         r = requests.get(url, stream=True)
@@ -132,12 +116,6 @@ def file_age(path):
     )
 
 
-def file_date_human(num):
-    return datetime.fromtimestamp(
-        num
-    ).strftime(app.config['DATE_FORMAT'])
-
-
 def file_name_version(full_path):
     """ New name versioned with date of the file  """
     file_dir = os.path.dirname(full_path)
@@ -188,20 +166,6 @@ def file_stat(path, filename):
     }
 
 
-def file_size_human(num, HTML = True):
-    space = ' ' if HTML else ' '
-    for x in [space + 'B', 'KB', 'MB', 'GB', 'TB']:
-        if num < 1024.0:
-            if x == space + 'B':
-                return "%d%s%s" % (num, space, x)
-            return "%3.1f%s%s" % (num, space, x)
-        num /= 1024.0
-
-
-def file_size_MB(num):
-    return "{:,.2f}".format(num/(1024*1024))
-
-
 def get_folder_size(path):
     total_size = 0
     for dirpath, dirnames, filenames in os.walk(path):
@@ -211,11 +175,24 @@ def get_folder_size(path):
     return total_size
 
 
-def get_or_none(key,d,none = None):
-    if key in d:
-        return d[key]
+def get_download_url(share, file, token):
+    direct = get_or_none('direct_links', share, False)
+    if direct:
+        return  "/".join((
+            app.config['PUBLIC_URL'],
+            'direct',
+            share['name'],
+            get_direct_token(share, file),
+            path2url(file)
+        ))
     else:
-        return none
+        return "/".join((
+            app.config['PUBLIC_URL'],
+            'download',
+            share['name'],
+            token,
+            path2url(file)
+        ))
 
 
 def get_script_url(public_url, share, end_point, token = "[TOKEN]"):
@@ -244,22 +221,6 @@ def get_script_url(public_url, share, end_point, token = "[TOKEN]"):
     return {'cmd': cmd, 'doc': doc}
 
 
-def is_path_safe(path):
-    if path.startswith("."):
-        return False
-    if "/." in path:
-        return False
-    return True
-
-
-def is_valid_url(url, qualifying = None):
-    min_attributes = ('scheme', 'netloc')
-    qualifying = min_attributes if qualifying is None else qualifying
-    token = urlparse(url)
-    return all([getattr(token, qualifying_attr)
-                for qualifying_attr in qualifying])
-
-
 def iter_folder_files(path, recursive = True, version_folder = None):
     if recursive:
         for dirpath, dirnames, filenames in os.walk(path, topdown = False):
@@ -287,10 +248,6 @@ def iter_folder_files(path, recursive = True, version_folder = None):
             yield fp
 
 
-def path2url(path):
-    return pathname2url(path)
-
-
 def read_config(app):
     # Read config from json
     config_values = json.load(open(os.getenv('FLEES_CONFIG'),'rt'))
@@ -318,22 +275,6 @@ def read_config(app):
     return config_values
 
 
-def safe_name(s):
-    return safe_string(s, "-_")
-
-
-def safe_path(s):
-    return safe_string(s, "-_/")
-
-
-def safe_string(s, valid, no_repeat = False):
-    """ return a safe string, replace non alnum characters with _ . all characters in valid are considered valid. """
-    safe = "".join([c if c.isalnum() or c in valid else "_" for c in s])
-    if no_repeat:
-        safe = re.sub(r'_+', '_', safe)
-    return safe
-
-
 def set_rights(path):
     os.chown(path, app.config['UID'], app.config['GID'])
     st = os.stat(path)

code/utils/misc.py

@@ -0,0 +1,88 @@
+from datetime import datetime
+from flask import current_app as app
+
+try:
+    from urllib.request import pathname2url
+    from urllib.request import urlparse
+except ImportError:
+    from urllib import pathname2url
+    from urlparse import urlparse
+
+try:
+    from werkzeug.utils import secure_filename
+except ImportError:
+    pass
+
+
+# String handling etc
+
+def bool_short(var):
+    if type(var) == bool:
+        if var:
+            return "Y"
+        else:
+            return "N"
+    return var
+
+
+def file_date_human(num):
+    return datetime.fromtimestamp(
+        num
+    ).strftime(app.config['DATE_FORMAT'])
+
+
+def file_size_human(num, HTML = True):
+    space = ' ' if HTML else ' '
+    for x in [space + 'B', 'KB', 'MB', 'GB', 'TB']:
+        if num < 1024.0:
+            if x == space + 'B':
+                return "%d%s%s" % (num, space, x)
+            return "%3.1f%s%s" % (num, space, x)
+        num /= 1024.0
+
+
+def file_size_MB(num):
+    return "{:,.2f}".format(num/(1024*1024))
+
+
+def get_or_none(key,d,none = None):
+    if key in d:
+        return d[key]
+    else:
+        return none
+
+
+def is_path_safe(path):
+    if path.startswith("."):
+        return False
+    if "/." in path:
+        return False
+    return True
+
+
+def is_valid_url(url, qualifying = None):
+    min_attributes = ('scheme', 'netloc')
+    qualifying = min_attributes if qualifying is None else qualifying
+    token = urlparse(url)
+    return all([getattr(token, qualifying_attr)
+                for qualifying_attr in qualifying])
+
+
+def path2url(path):
+    return pathname2url(path)
+
+
+def safe_name(s):
+    return safe_string(s, "-_")
+
+
+def safe_path(s):
+    return safe_string(s, "-_/")
+
+
+def safe_string(s, valid, no_repeat = False):
+    """ return a safe string, replace non alnum characters with _ . all characters in valid are considered valid. """
+    safe = "".join([c if c.isalnum() or c in valid else "_" for c in s])
+    if no_repeat:
+        safe = re.sub(r'_+', '_', safe)
+    return safe