From 650df3f2048041612212490f2762aa8db828f9e0 Mon Sep 17 00:00:00 2001 From: Ville Rantanen Date: Sun, 25 Feb 2018 20:32:02 +0200 Subject: [PATCH] big rewrite of token system --- README.md | 33 ++-- code/Dockerfile | 2 +- code/app.py | 147 +++++++----------- ...quirements.txt => docker-requirements.txt} | 0 {utils => code}/flees-manager.py | 85 ++++------ .../manager-requirements.txt | 0 code/templates/list.html | 10 +- code/utils/__init__.py | 1 + code/utils/crypt.py | 58 +++++++ code/utils/utils.py | 49 ++++++ 10 files changed, 218 insertions(+), 167 deletions(-) rename code/{requirements.txt => docker-requirements.txt} (100%) rename {utils => code}/flees-manager.py (90%) rename utils/requirements.txt => code/manager-requirements.txt (100%) create mode 100644 code/utils/__init__.py create mode 100644 code/utils/crypt.py create mode 100644 code/utils/utils.py diff --git a/README.md b/README.md index b535c54..68d40df 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,18 @@ The name comes from mispronouncing "files" very badly. - `touch code/notifier.py` - `docker-compose up --build` - open URL: http://localhost:8136/list/test +- `pip install code/manager-requirements.txt` +# configuration - -- configure shares with data/shares.json -- generate and manage shares with utils/flees-manager.py +- generate and manage shares with `code/flees-manager.py` - configure service with data/config.json + - Change your app_secret_key !! + - Change your public_url - uid = user id for new files - workers = parallel processes (i.e. one upload reserves a process) - timeout = timeout for processes, single upload might take a long time! + - max_zip_size = zipping a share with more data is not allowed - configure bind host and port in .env - proxy with nginx, match body size and timeout to your needs: ``` @@ -38,15 +41,8 @@ location /flees/ { - configure local port in `docker-compose.yaml` -- directly login with URLs: - - http://host/list/[share name]/[hashed password] -- download with curl (etc.) - - http://host/download/[share name]/[hashed password]/[filename] -- upload with curl (etc.) - - curl -F file=@my.file http://host/upload/[share name]/[hashed password] -- "direct link" is a sharing link that does not require other passwords, and is unique to each file. - (there should be no danger in sharing a file, and the password to rest of the files leaking) - +- Check `flees-manager.py rest` command to get direct links to various + actions # custom notifier @@ -62,14 +58,21 @@ Flees will send notification on upload and download events, with a Dict like thi "recipient": "share recipient", "share": "name", "filename": "file_path", - "operation": "direct_download" + "operation": "direct_download", + "environment": [env for request, including IP addresses etc] } ``` Operation is one of download, direct_download, zip_download, or upload +# Passwords - - +- shares.json stores hashed version of password. +- Additionally, it may store plain text password, if users so wish. +- Internally, Flees only compares the hashes of passwords +- Tokens are encrypted versions of the hash. (login/upload/download with + direct links). i.e. decrypted URL request equals password hash +- Encryption key is the app_secret_key +- Direct download token is (password hash + filename) hashed diff --git a/code/Dockerfile b/code/Dockerfile index 7925247..fe01498 100644 --- a/code/Dockerfile +++ b/code/Dockerfile @@ -4,7 +4,7 @@ RUN apk add --update \ python3-dev \ py3-pip \ build-base -COPY requirements.txt /requirements.txt +COPY docker-requirements.txt /requirements.txt RUN pip3 install -r /requirements.txt COPY static /code/static COPY templates /code/templates diff --git a/code/app.py b/code/app.py index b4e3a27..c361818 100644 --- a/code/app.py +++ b/code/app.py @@ -11,8 +11,10 @@ import hashlib import zipfile from multiprocessing import Process from revprox import ReverseProxied +from utils.utils import * +from utils.crypt import * -__FLEES_VERSION__ = "20180224.0b" +__FLEES_VERSION__ = "20180225.0" app = Flask(__name__) app.config.from_object(__name__) # Read config from json ! @@ -39,6 +41,7 @@ if 'notifier' in config_values: app.secret_key = config_values['app_secret_key'] app.wsgi_app = ReverseProxied(app.wsgi_app) +app.config['CRYPTO'] = Crypto(app.secret_key) @app.before_request def before_request(): @@ -51,7 +54,7 @@ def before_request(): def index(): public_shares = [] for share in g.shares: - public = get_or_none(share,'public') + public = get_or_none('public', share) expired = is_expired(share) authenticated_share = get_share(share['name']) password_set = False @@ -61,10 +64,10 @@ def index(): if public or password_set: public_shares.append({ 'name': share['name'], - 'expire': get_or_none(share,'expire'), - 'upload': get_or_none(share,'upload'), + 'expire': get_or_none('expire', share), + 'upload': get_or_none('upload', share), 'password_set': password_set, - 'description': get_or_none(share,'description','') + 'description': get_or_none('description', share, '') }) return render_template("index.html", entries=public_shares) @@ -75,7 +78,7 @@ def authenticate(name): return render_template('authenticate.html',name=name) if request.method == 'POST': user_password = request.form['password'].encode('utf-8') - session[name] = hashlib.sha1(user_password).hexdigest() + session[name] = password_hash(user_password) return redirect(url_for('list_view',name=name)) @app.route('/upload//', methods=['POST']) @@ -86,11 +89,11 @@ def upload(name = None, password = None): if name == None: name = request.form['name'] if password != None: - session[name] = password + session[name] = app.config['CRYPTO'].decrypt(password) (ok,share) = get_share(name) if not ok: return share - if not get_or_none(share,'upload') == True: + if not get_or_none('upload', share) == True: return "Upload not allowed\n",400 if file: filename = os.path.join( @@ -99,14 +102,14 @@ def upload(name = None, password = None): file.filename ) ) - if get_or_none(share, 'overwrite') == False: + if get_or_none('overwrite', share) == False: if os.path.exists(filename): file_versionize(filename) #~ return "Overwrite forbidden", 403 file.save(filename) set_rights(filename) notify({ - "recipient": get_or_none(share,'recipient'), + "recipient": get_or_none('recipient', share), "share": name, "filename": filename, "operation": "upload" @@ -122,11 +125,11 @@ def upload(name = None, password = None): @app.route('/upload_join//', methods=['POST']) def upload_join_splitted(name, password): if request.method == 'POST': - session[name] = password + session[name] = app.config['CRYPTO'].decrypt(password) (ok,share) = get_share(name) if not ok: return share - if not get_or_none(share,'upload') == True: + if not get_or_none('upload', share) == True: return "Upload not allowed",400 if not 'filename' in request.form: return "No filename given", 400 @@ -150,7 +153,7 @@ def upload_join_splitted(name, password): share['path'], request.form['filename'] ) - if get_or_none(share, 'overwrite') == False: + if get_or_none('overwrite', share) == False: if os.path.exists(target_name): file_versionize(target_name) @@ -175,7 +178,8 @@ def send(name): @app.route('/list/', methods=['GET']) def list_view(name, password = None): if password != None: - session[name] = password + session[name] = app.config['CRYPTO'].decrypt(password) + return redirect(url_for('list_view',name=name)) (ok,share) = get_share(name) if not ok: return share @@ -192,22 +196,26 @@ def list_view(name, password = None): }) files.append(status) # direct share links not allowed if password isnt set - allow_direct = get_or_none(share,'direct_links') if get_or_none(share,'pass_hash') else False + allow_direct = get_or_none('direct_links', share) if get_or_none('pass_hash', share) else False + upload = get_or_none('upload', share) + overwrite = get_or_none('overwrite', share) + if not upload: + overwrite = False return render_template( "list.html", name = share['name'], entries = files, - password = get_or_none(share,'pass_hash'), - public = get_or_none(share,'public'), - upload = get_or_none(share,'upload'), - overwrite = get_or_none(share,'overwrite'), + password = get_or_none('pass_hash', share), + public = get_or_none('public', share), + upload = upload, + overwrite = overwrite, direct = allow_direct, - expire = get_or_none(share,'expire'), - description = get_or_none(share,'description',"") + expire = get_or_none('expire', share), + description = get_or_none('description', share, "") ) @app.route('/logout/', methods=['GET']) -def logout(name, password = None): +def logout(name): if name in session: del session[name] return render_template( @@ -215,26 +223,24 @@ def logout(name, password = None): name = name ) -@app.route('/direct///', methods=['GET']) -def download_direct(name,password,filename): - if password != None: - session[name] = password +@app.route('/direct///', methods=['GET']) +def download_direct(name,token,filename): (ok,share) = get_share(name, require_auth = False) if not ok: return share - allow_direct = get_or_none(share,'direct_links') + allow_direct = get_or_none('direct_links', share) if allow_direct != True: return 'Direct download not allowed', 403 - token = get_direct_token(share, filename) - if token == None: + file_token = get_direct_token(share, filename) + if file_token == None: return 'Cannot generate token', 400 - if password != token: + if file_token != token: return 'Incorrect token', 403 file_path = os.path.join(share['path'], filename) if not os.path.exists(file_path): return 'no such file', 404 notify({ - "recipient": get_or_none(share,'recipient'), + "recipient": get_or_none('recipient', share), "share": name, "filename": file_path, "operation": "direct_download" @@ -246,7 +252,7 @@ def download_direct(name,password,filename): @app.route('/download//', methods=['GET']) def download_file(name,filename,password = None): if password != None: - session[name] = password + session[name] = app.config['CRYPTO'].decrypt(password) (ok,share) = get_share(name) if not ok: return share @@ -254,7 +260,7 @@ def download_file(name,filename,password = None): if not os.path.exists(file_path): return 'no such file', 404 notify({ - "recipient": get_or_none(share,'recipient'), + "recipient": get_or_none('recipient', share), "share": name, "filename": file_path, "operation": "download" @@ -266,7 +272,7 @@ def download_file(name,filename,password = None): @app.route('/zip/', methods=['GET']) def download_zip(name,password = None): if password != None: - session[name] = password + session[name] = app.config['CRYPTO'].decrypt(password) (ok,share) = get_share(name) if not ok: return share @@ -276,7 +282,7 @@ def download_zip(name,password = None): zip_clean() zip_path = zip_share(share) notify({ - "recipient": get_or_none(share,'recipient'), + "recipient": get_or_none('recipient', share), "share": name, "filename": name + ".zip", "operation": "zip_download" @@ -289,11 +295,11 @@ def download_zip(name,password = None): @app.route('/script/upload//', methods=['GET']) def script_upload(name = None, password = None): - session[name] = password + session[name] = app.config['CRYPTO'].decrypt(password) (ok,share) = get_share(name) if not ok: return share - if not get_or_none(share,'upload') == True: + if not get_or_none('upload', share) == True: return "Upload not allowed",400 return """#!/bin/bash test -n "$1" || { @@ -336,7 +342,7 @@ done @app.route('/script/download//', methods=['GET']) def script_download(name = None, password = None): - session[name] = password + session[name] = app.config['CRYPTO'].decrypt(password) (ok,share) = get_share(name) if not ok: return share @@ -392,7 +398,7 @@ get_file() { @app.route('/script/direct//', methods=['GET']) def script_direct(name = None, password = None): - session[name] = password + session[name] = app.config['CRYPTO'].decrypt(password) (ok,share) = get_share(name) if not ok: return share @@ -448,11 +454,11 @@ get_file() { @app.route('/script/upload_split//', methods=['GET']) def script_upload_split(name = None, password = None): - session[name] = password + session[name] = app.config['CRYPTO'].decrypt(password) (ok,share) = get_share(name) if not ok: return share - if not get_or_none(share,'upload') == True: + if not get_or_none('upload', share) == True: return "Upload not allowed",400 return """#!/bin/bash test -n "$1" || { @@ -527,31 +533,6 @@ class uploadJoiner: for part in self.parts: os.remove(part) -def file_stat(filename): - s = os.stat(filename) - return { - 'size': file_size_MB(s.st_size), - 'mtime': file_date_human(s.st_mtime), - 'name': os.path.basename(filename) - } - - -def file_size_human(num): - for x in [' B','KB','MB','GB','TB']: - if num < 1024.0: - if x==' B': - return "%d %s" % (num, x) - return "%3.1f %s" % (num, x) - num /= 1024.0 - -def file_size_MB(num): - return "{:,.2f}".format(num/(1024*1024)) - - -def file_date_human(num): - return datetime.fromtimestamp( - num - ).strftime(app.config['DATE_FORMAT']) def file_versionize(filename): """ Move file to versioned with integer """ @@ -574,33 +555,6 @@ def file_versionize(filename): os.rename(filename,new_name) -def get_direct_token(share, filename): - if not 'pass_hash' in share: - return None - return hashlib.sha1( - share['pass_hash'].encode('utf-8') + filename.encode('utf-8') - ).hexdigest() - -def get_folder_size(path): - - total_size = 0 - for filename in os.listdir(path): - fp = os.path.join(path, filename) - if os.path.isdir(fp): - continue - total_size += os.path.getsize( - fp - ) - return total_size - - -def get_or_none(d,key,none = None): - if key in d: - return d[key] - else: - return none - - def get_share(name, require_auth = True): share = [x for x in g.shares if x['name'] == name] if len(share) < 1: @@ -632,17 +586,19 @@ def get_share(name, require_auth = True): return (True,share) def is_expired(share): - expires = get_or_none(share, 'expire') + expires = get_or_none('expire', share) if expires: if datetime.now() > datetime.strptime(expires, app.config['DATE_FORMAT']): return True return False + def print_debug(s): if app.config['DEBUG']: sys.stderr.write(str(s)+"\n") sys.stderr.flush() + def makedirs_rights(path): # os.makedirs with chown path_list = path.split(os.sep) @@ -667,7 +623,6 @@ def set_rights(path): os.chmod(path, st.st_mode | stat.S_IRGRP | stat.S_IWGRP) - def zip_share(share): if not os.path.exists(app.config['ZIP_FOLDER']): @@ -695,6 +650,7 @@ def zip_share(share): set_rights(zip_path) return zip_path + def zip_clean(): """ delete zip files older than 1 hour """ if not os.path.exists(app.config['ZIP_FOLDER']): @@ -708,6 +664,7 @@ def zip_clean(): if mtime + 3600 < time.time(): os.remove(os.path.join(app.config['ZIP_FOLDER'],file)) + if __name__ == "__main__": app.run(debug=True) diff --git a/code/requirements.txt b/code/docker-requirements.txt similarity index 100% rename from code/requirements.txt rename to code/docker-requirements.txt diff --git a/utils/flees-manager.py b/code/flees-manager.py similarity index 90% rename from utils/flees-manager.py rename to code/flees-manager.py index 207f193..076a4b1 100755 --- a/utils/flees-manager.py +++ b/code/flees-manager.py @@ -2,30 +2,9 @@ import argparse,json,sys,os from shutil import copyfile from tabulate import tabulate -import hashlib from datetime import datetime - -def get_folder_size(path): - - total_size = 0 - for dirpath, dirnames, filenames in os.walk(path): - for f in filenames: - fp = os.path.join(dirpath, f) - total_size += os.path.getsize(fp) - return total_size - -def get_or_no(key,d,no): - if key in d: - return d[key] - return no - -def get_direct_token(share, filename): - if not 'pass_hash' in share: - return None - return hashlib.sha1( - share['pass_hash'].encode('utf-8') + filename.encode('utf-8') - ).hexdigest() - +from utils.utils import * +from utils.crypt import * def get_root_path(opts): root_folder = os.path.dirname( @@ -37,19 +16,12 @@ def get_root_path(opts): ) return root_folder -def file_size_human(num): - for x in ['B','KB','MB','GB','TB']: - if num < 1024.0: - if x=='B': - return "%d %s" % (num, x) - return "%3.1f %s" % (num, x) - num /= 1024.0 def list_shares(shares,opts): table = [] table.append(('Name', 'Path','Public','Password','PassHash','Upload','Overwrite','Direct','Expire','Recipient','Description')) for share in shares: - public = get_or_no('public',share, False) + public = get_or_none('public',share, False) passhash = '-' password = 'pass_hash' in share if opts.show_password: @@ -61,11 +33,11 @@ def list_shares(shares,opts): passhash = share['pass_hash'] else: passhash = "-" - upload = get_or_no('upload',share, False) - overwrite = get_or_no('overwrite',share, True) - direct = get_or_no('direct_links',share, False) if password else False - expire = get_or_no('expire',share, "-") - description = get_or_no('description',share, "")[0:20] + upload = get_or_none('upload',share, False) + overwrite = get_or_none('overwrite',share, True) + direct = get_or_none('direct_links',share, False) if password else False + expire = get_or_none('expire',share, "-") + description = get_or_none('description',share, "")[0:20] table.append(( share['name'], share['path']+"/", @@ -76,7 +48,7 @@ def list_shares(shares,opts): overwrite, direct, expire, - get_or_no('recipient', share, "")[0:20], + get_or_none('recipient', share, "")[0:20], description )) print(tabulate(table, headers = "firstrow")) @@ -104,7 +76,7 @@ def list_folders(shares,config): break (size_num, size_unit) = file_size_human(get_folder_size( full_path - )).split(" ") + )).split(" ") table.append(( folder, share_name, @@ -129,7 +101,7 @@ def add_share(shares, config, opts): if opts.password: if opts.plain: share['pass_plain'] = opts.password - share['pass_hash'] = hashlib.sha1(opts.password).hexdigest() + share['pass_hash'] = password_hash(opts.password) if opts.expire: try: date_object = datetime.strptime(opts.expire,"%Y-%m-%d %H:%M") @@ -193,7 +165,7 @@ def modify_share(shares, config, opts): # ADD/Change a password if opts.plain: share['pass_plain'] = opts.password - share['pass_hash'] = hashlib.sha1(opts.password).hexdigest() + share['pass_hash'] = password_hash(opts.password) if opts.expire: if opts.expire == "": @@ -263,7 +235,6 @@ def print_rest_api(shares, config, opts): if 'public_url' not in config: print("Set public_url variable in your config.json") sys.exit(1) - shares = [share for share in shares if share['name'] == opts.name] if len(shares) == 0: print("No such share %s"%( opts.name, )) @@ -271,7 +242,7 @@ def print_rest_api(shares, config, opts): share = shares[0] if opts.type == "list": - print("Link to list contents of the share:") + print("Link to enter the share:") print("%s/list/%s"%( config['public_url'], share['name'] @@ -280,12 +251,14 @@ def print_rest_api(shares, config, opts): if not 'pass_hash' in share: print("REST API enabled only if pass_hash is set for share") sys.exit(1) + crypter = Crypto(config['app_secret_key']) + crypted = crypter.encrypt(share['pass_hash']) if opts.type == "login": print("Link to automatically login in the share:") print("%s/list/%s/%s"%( config['public_url'], share['name'], - share['pass_hash'] + crypted )) elif opts.type == "upload": if 'upload' not in share or not share['upload']: @@ -293,15 +266,22 @@ def print_rest_api(shares, config, opts): sys.exit(0) print("Link to upload file to the share:") - print("\n# curl -F file=@'the_file_name.ext' %s/upload/%s/%s"%( + print("\n# curl -F file=@'the_file_name.ext' %s/upload/%s/%s"%( config['public_url'], share['name'], - share['pass_hash'] + crypted )) - print("or \n\n# curl -s %s/script/upload/%s/%s | bash /dev/stdin file_to_upload.ext [second.file.ext]"%( + print("\nLink to upload multiple files to the share:") + print("\n# curl -s %s/script/upload/%s/%s | bash /dev/stdin file_to_upload.ext [second.file.ext]"%( config['public_url'], share['name'], - share['pass_hash'] + crypted + )) + print("\nLink to upload multiple files to the share, splitting large files (default 512Mb):") + print("\n# curl -s %s/script/upload_split/%s/%s | bash /dev/stdin [-s split_size_in_Mb] file_to_upload.ext [second.file.ext]"%( + config['public_url'], + share['name'], + crypted )) elif opts.type == "download": print("Links to download files:") @@ -321,13 +301,13 @@ def print_rest_api(shares, config, opts): print("%s/download/%s/%s/%s"%( config['public_url'], share['name'], - share['pass_hash'], + crypted, filename )) print("or \n\n# curl -s %s/script/download/%s/%s | bash /dev/stdin [-f]"%( config['public_url'], share['name'], - share['pass_hash'] + crypted )) elif opts.type == "direct": if 'direct_links' not in share or not share['direct_links']: @@ -356,18 +336,17 @@ def print_rest_api(shares, config, opts): print("or \n\n# curl -s %s/script/direct/%s/%s | bash /dev/stdin [-f]"%( config['public_url'], share['name'], - share['pass_hash'] + crypted )) elif opts.type == "zip": print("ZIP download:") print("%s/zip/%s/%s"%( config['public_url'], share['name'], - share['pass_hash'] + crypted )) - def parse_options(): config_default = os.path.realpath( os.path.join( @@ -390,7 +369,7 @@ def parse_options(): ## list shares parser_list = subparsers.add_parser('list', help = "List shares") parser_list.add_argument('-P', action="store_true", dest="show_password", default = False, - help = "Display hashed passwords") + help = "Display passwords") ## list folders parser_folders = subparsers.add_parser('folders', help = "List folders and share names") ## Remove diff --git a/utils/requirements.txt b/code/manager-requirements.txt similarity index 100% rename from utils/requirements.txt rename to code/manager-requirements.txt diff --git a/code/templates/list.html b/code/templates/list.html index 13e0bcb..4cae244 100644 --- a/code/templates/list.html +++ b/code/templates/list.html @@ -31,8 +31,12 @@ {% else %}
  • never expires {% endif %} - {% if overwrite == false %} -
  • overwriting is disabled + {% if upload %} + {% if overwrite %} +
  • uploads overwrite + {% else %} +
  • uploads versioned + {% endif %} {% endif %}
  • Download as zip
  • Logout @@ -54,7 +58,7 @@ {% if direct %} - ❖ + ❖ {% endif %} {{ entry.name }} {{ entry.size|safe }} diff --git a/code/utils/__init__.py b/code/utils/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/code/utils/__init__.py @@ -0,0 +1 @@ + diff --git a/code/utils/crypt.py b/code/utils/crypt.py new file mode 100644 index 0000000..4ee9b7d --- /dev/null +++ b/code/utils/crypt.py @@ -0,0 +1,58 @@ +import base64 +from Crypto.Cipher import AES +import hashlib + +class Crypto: + def __init__(self, secret): + self.secret = add_pad(secret[0:16]) + self.cipher = AES.new(self.secret, AES.MODE_ECB) + + def encrypt(self, msg): + + return base64.urlsafe_b64encode( + self.cipher.encrypt( + add_pad( + msg + ) + ) + ).decode("utf-8") + + + def decrypt(self, enc): + + return remove_pad( + self.cipher.decrypt( + base64.urlsafe_b64decode( + enc + ) + ).decode("utf-8") + ) + + +def add_pad(string): + """ Add spaces until length is multiple of 16 """ + while len(string)%16 != 0: + string+=" " + return string + + +def get_direct_token(share, filename): + if not 'pass_hash' in share: + return None + return password_hash( + share['pass_hash'] + filename + ) + + +def password_hash(string): + if type(string) == str: + string = string.encode("utf-8") + return hashlib.sha1( + string + ).hexdigest() + + +def remove_pad(string): + """ Remove spaces from right """ + return string.rstrip(" ") + diff --git a/code/utils/utils.py b/code/utils/utils.py new file mode 100644 index 0000000..ec0bf54 --- /dev/null +++ b/code/utils/utils.py @@ -0,0 +1,49 @@ +import os +import hashlib +from datetime import datetime +from flask import current_app as app + + +def file_date_human(num): + return datetime.fromtimestamp( + num + ).strftime(app.config['DATE_FORMAT']) + + +def file_stat(filename): + s = os.stat(filename) + return { + 'size': file_size_MB(s.st_size), + 'mtime': file_date_human(s.st_mtime), + 'name': os.path.basename(filename) + } + + +def file_size_human(num,HTML=True): + space = ' ' if HTML else ' ' + for x in [space + 'B', 'KB', 'MB', 'GB', 'TB']: + if num < 1024.0: + if x == space + 'B': + return "%d%s%s" % (num, space, x) + return "%3.1f%s%s" % (num, space, x) + num /= 1024.0 + + +def file_size_MB(num): + return "{:,.2f}".format(num/(1024*1024)) + + +def get_folder_size(path): + total_size = 0 + for dirpath, dirnames, filenames in os.walk(path): + for f in filenames: + fp = os.path.join(dirpath, f) + total_size += os.path.getsize(fp) + return total_size + + +def get_or_none(key,d,none = None): + if key in d: + return d[key] + else: + return none