moonq/flees
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

big rewrite of token system

Browse Source
This commit is contained in:
Ville Rantanen
2018-02-25 20:32:02 +02:00
parent 525070fe23
commit 650df3f204
10 changed files with 218 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
README.md
View File

@@ -13,15 +13,18 @@ The name comes from mispronouncing "files" very badly.
- `touch code/notifier.py`
- `docker-compose up --build`
- open URL: http://localhost:8136/list/test
- `pip install code/manager-requirements.txt`
# configuration
- configure shares with data/shares.json
- generate and manage shares with utils/flees-manager.py
- generate and manage shares with `code/flees-manager.py`
- configure service with data/config.json
- Change your app_secret_key !!
- Change your public_url
- uid = user id for new files
- workers = parallel processes (i.e. one upload reserves a process)
- timeout = timeout for processes, single upload might take a long time!
- max_zip_size = zipping a share with more data is not allowed
- configure bind host and port in .env
- proxy with nginx, match body size and timeout to your needs:
```
@@ -38,15 +41,8 @@ location /flees/ {
- configure local port in `docker-compose.yaml`
- directly login with URLs:
- http://host/list/[share name]/[hashed password]
- download with curl (etc.)
- http://host/download/[share name]/[hashed password]/[filename]
- upload with curl (etc.)
- curl -F file=@my.file http://host/upload/[share name]/[hashed password]
- "direct link" is a sharing link that does not require other passwords, and is unique to each file.
(there should be no danger in sharing a file, and the password to rest of the files leaking)
- Check `flees-manager.py rest` command to get direct links to various
actions
# custom notifier
@@ -62,14 +58,21 @@ Flees will send notification on upload and download events, with a Dict like thi
"recipient": "share recipient",
"share": "name",
"filename": "file_path",
"operation": "direct_download"
"operation": "direct_download",
"environment": [env for request, including IP addresses etc]
}
```
Operation is one of download, direct_download, zip_download, or upload
# Passwords
- shares.json stores hashed version of password.
- Additionally, it may store plain text password, if users so wish.
- Internally, Flees only compares the hashes of passwords
- Tokens are encrypted versions of the hash. (login/upload/download with
direct links). i.e. decrypted URL request equals password hash
- Encryption key is the app_secret_key
- Direct download token is (password hash + filename) hashed

2
code/Dockerfile
View File

@@ -4,7 +4,7 @@ RUN apk add --update \
python3-dev \
py3-pip \
build-base
COPY requirements.txt /requirements.txt
COPY docker-requirements.txt /requirements.txt
RUN pip3 install -r /requirements.txt
COPY static /code/static
COPY templates /code/templates

147
code/app.py
View File

@@ -11,8 +11,10 @@ import hashlib
import zipfile
from multiprocessing import Process
from revprox import ReverseProxied
from utils.utils import *
from utils.crypt import *
__FLEES_VERSION__ = "20180224.0b"
__FLEES_VERSION__ = "20180225.0"
app = Flask(__name__)
app.config.from_object(__name__)
# Read config from json !
@@ -39,6 +41,7 @@ if 'notifier' in config_values:
app.secret_key = config_values['app_secret_key']
app.wsgi_app = ReverseProxied(app.wsgi_app)
app.config['CRYPTO'] = Crypto(app.secret_key)
@app.before_request
def before_request():
@@ -51,7 +54,7 @@ def before_request():
def index():
public_shares = []
for share in g.shares:
public = get_or_none(share,'public')
public = get_or_none('public', share)
expired = is_expired(share)
authenticated_share = get_share(share['name'])
password_set = False
@@ -61,10 +64,10 @@ def index():
if public or password_set:
public_shares.append({
'name': share['name'],
'expire': get_or_none(share,'expire'),
'upload': get_or_none(share,'upload'),
'expire': get_or_none('expire', share),
'upload': get_or_none('upload', share),
'password_set': password_set,
'description': get_or_none(share,'description','')
'description': get_or_none('description', share, '')
})
return render_template("index.html", entries=public_shares)
@@ -75,7 +78,7 @@ def authenticate(name):
return render_template('authenticate.html',name=name)
if request.method == 'POST':
user_password = request.form['password'].encode('utf-8')
session[name] = hashlib.sha1(user_password).hexdigest()
session[name] = password_hash(user_password)
return redirect(url_for('list_view',name=name))
@app.route('/upload/<name>/<password>', methods=['POST'])
@@ -86,11 +89,11 @@ def upload(name = None, password = None):
if name == None:
name = request.form['name']
if password != None:
session[name] = password
session[name] = app.config['CRYPTO'].decrypt(password)
(ok,share) = get_share(name)
if not ok:
return share
if not get_or_none(share,'upload') == True:
if not get_or_none('upload', share) == True:
return "Upload not allowed\n",400
if file:
filename = os.path.join(
@@ -99,14 +102,14 @@ def upload(name = None, password = None):
file.filename
)
)
if get_or_none(share, 'overwrite') == False:
if get_or_none('overwrite', share) == False:
if os.path.exists(filename):
file_versionize(filename)
#~ return "Overwrite forbidden", 403
file.save(filename)
set_rights(filename)
notify({
"recipient": get_or_none(share,'recipient'),
"recipient": get_or_none('recipient', share),
"share": name,
"filename": filename,
"operation": "upload"
@@ -122,11 +125,11 @@ def upload(name = None, password = None):
@app.route('/upload_join/<name>/<password>', methods=['POST'])
def upload_join_splitted(name, password):
if request.method == 'POST':
session[name] = password
session[name] = app.config['CRYPTO'].decrypt(password)
(ok,share) = get_share(name)
if not ok:
return share
if not get_or_none(share,'upload') == True:
if not get_or_none('upload', share) == True:
return "Upload not allowed",400
if not 'filename' in request.form:
return "No filename given", 400
@@ -150,7 +153,7 @@ def upload_join_splitted(name, password):
share['path'],
request.form['filename']
)
if get_or_none(share, 'overwrite') == False:
if get_or_none('overwrite', share) == False:
if os.path.exists(target_name):
file_versionize(target_name)
@@ -175,7 +178,8 @@ def send(name):
@app.route('/list/<name>', methods=['GET'])
def list_view(name, password = None):
if password != None:
session[name] = password
session[name] = app.config['CRYPTO'].decrypt(password)
return redirect(url_for('list_view',name=name))
(ok,share) = get_share(name)
if not ok:
return share
@@ -192,22 +196,26 @@ def list_view(name, password = None):
})
files.append(status)
# direct share links not allowed if password isnt set
allow_direct = get_or_none(share,'direct_links') if get_or_none(share,'pass_hash') else False
allow_direct = get_or_none('direct_links', share) if get_or_none('pass_hash', share) else False
upload = get_or_none('upload', share)
overwrite = get_or_none('overwrite', share)
if not upload:
overwrite = False
return render_template(
"list.html",
name = share['name'],
entries = files,
password = get_or_none(share,'pass_hash'),
public = get_or_none(share,'public'),
upload = get_or_none(share,'upload'),
overwrite = get_or_none(share,'overwrite'),
password = get_or_none('pass_hash', share),
public = get_or_none('public', share),
upload = upload,
overwrite = overwrite,
direct = allow_direct,
expire = get_or_none(share,'expire'),
description = get_or_none(share,'description',"")
expire = get_or_none('expire', share),
description = get_or_none('description', share, "")
)
@app.route('/logout/<name>', methods=['GET'])
def logout(name, password = None):
def logout(name):
if name in session:
del session[name]
return render_template(
@@ -215,26 +223,24 @@ def logout(name, password = None):
name = name
)
@app.route('/direct/<name>/<password>/<filename>', methods=['GET'])
def download_direct(name,password,filename):
if password != None:
session[name] = password
@app.route('/direct/<name>/<token>/<filename>', methods=['GET'])
def download_direct(name,token,filename):
(ok,share) = get_share(name, require_auth = False)
if not ok:
return share
allow_direct = get_or_none(share,'direct_links')
allow_direct = get_or_none('direct_links', share)
if allow_direct != True:
return 'Direct download not allowed', 403
token = get_direct_token(share, filename)
if token == None:
file_token = get_direct_token(share, filename)
if file_token == None:
return 'Cannot generate token', 400
if password != token:
if file_token != token:
return 'Incorrect token', 403
file_path = os.path.join(share['path'], filename)
if not os.path.exists(file_path):
return 'no such file', 404
notify({
"recipient": get_or_none(share,'recipient'),
"recipient": get_or_none('recipient', share),
"share": name,
"filename": file_path,
"operation": "direct_download"
@@ -246,7 +252,7 @@ def download_direct(name,password,filename):
@app.route('/download/<name>/<filename>', methods=['GET'])
def download_file(name,filename,password = None):
if password != None:
session[name] = password
session[name] = app.config['CRYPTO'].decrypt(password)
(ok,share) = get_share(name)
if not ok:
return share
@@ -254,7 +260,7 @@ def download_file(name,filename,password = None):
if not os.path.exists(file_path):
return 'no such file', 404
notify({
"recipient": get_or_none(share,'recipient'),
"recipient": get_or_none('recipient', share),
"share": name,
"filename": file_path,
"operation": "download"
@@ -266,7 +272,7 @@ def download_file(name,filename,password = None):
@app.route('/zip/<name>', methods=['GET'])
def download_zip(name,password = None):
if password != None:
session[name] = password
session[name] = app.config['CRYPTO'].decrypt(password)
(ok,share) = get_share(name)
if not ok:
return share
@@ -276,7 +282,7 @@ def download_zip(name,password = None):
zip_clean()
zip_path = zip_share(share)
notify({
"recipient": get_or_none(share,'recipient'),
"recipient": get_or_none('recipient', share),
"share": name,
"filename": name + ".zip",
"operation": "zip_download"
@@ -289,11 +295,11 @@ def download_zip(name,password = None):
@app.route('/script/upload/<name>/<password>', methods=['GET'])
def script_upload(name = None, password = None):
session[name] = password
session[name] = app.config['CRYPTO'].decrypt(password)
(ok,share) = get_share(name)
if not ok:
return share
if not get_or_none(share,'upload') == True:
if not get_or_none('upload', share) == True:
return "Upload not allowed",400
return """#!/bin/bash
test -n "$1" || {
@@ -336,7 +342,7 @@ done
@app.route('/script/download/<name>/<password>', methods=['GET'])
def script_download(name = None, password = None):
session[name] = password
session[name] = app.config['CRYPTO'].decrypt(password)
(ok,share) = get_share(name)
if not ok:
return share
@@ -392,7 +398,7 @@ get_file() {
@app.route('/script/direct/<name>/<password>', methods=['GET'])
def script_direct(name = None, password = None):
session[name] = password
session[name] = app.config['CRYPTO'].decrypt(password)
(ok,share) = get_share(name)
if not ok:
return share
@@ -448,11 +454,11 @@ get_file() {
@app.route('/script/upload_split/<name>/<password>', methods=['GET'])
def script_upload_split(name = None, password = None):
session[name] = password
session[name] = app.config['CRYPTO'].decrypt(password)
(ok,share) = get_share(name)
if not ok:
return share
if not get_or_none(share,'upload') == True:
if not get_or_none('upload', share) == True:
return "Upload not allowed",400
return """#!/bin/bash
test -n "$1" || {
@@ -527,31 +533,6 @@ class uploadJoiner:
for part in self.parts:
os.remove(part)
def file_stat(filename):
s = os.stat(filename)
return {
'size': file_size_MB(s.st_size),
'mtime': file_date_human(s.st_mtime),
'name': os.path.basename(filename)
}
def file_size_human(num):
for x in ['&nbsp;B','KB','MB','GB','TB']:
if num < 1024.0:
if x=='&nbsp;B':
return "%d&nbsp;%s" % (num, x)
return "%3.1f&nbsp;%s" % (num, x)
num /= 1024.0
def file_size_MB(num):
return "{:,.2f}".format(num/(1024*1024))
def file_date_human(num):
return datetime.fromtimestamp(
num
).strftime(app.config['DATE_FORMAT'])
def file_versionize(filename):
""" Move file to versioned with integer """
@@ -574,33 +555,6 @@ def file_versionize(filename):
os.rename(filename,new_name)
def get_direct_token(share, filename):
if not 'pass_hash' in share:
return None
return hashlib.sha1(
share['pass_hash'].encode('utf-8') + filename.encode('utf-8')
).hexdigest()
def get_folder_size(path):
total_size = 0
for filename in os.listdir(path):
fp = os.path.join(path, filename)
if os.path.isdir(fp):
continue
total_size += os.path.getsize(
fp
)
return total_size
def get_or_none(d,key,none = None):
if key in d:
return d[key]
else:
return none
def get_share(name, require_auth = True):
share = [x for x in g.shares if x['name'] == name]
if len(share) < 1:
@@ -632,17 +586,19 @@ def get_share(name, require_auth = True):
return (True,share)
def is_expired(share):
expires = get_or_none(share, 'expire')
expires = get_or_none('expire', share)
if expires:
if datetime.now() > datetime.strptime(expires, app.config['DATE_FORMAT']):
return True
return False
def print_debug(s):
if app.config['DEBUG']:
sys.stderr.write(str(s)+"\n")
sys.stderr.flush()
def makedirs_rights(path):
# os.makedirs with chown
path_list = path.split(os.sep)
@@ -667,7 +623,6 @@ def set_rights(path):
os.chmod(path, st.st_mode | stat.S_IRGRP | stat.S_IWGRP)
def zip_share(share):
if not os.path.exists(app.config['ZIP_FOLDER']):
@@ -695,6 +650,7 @@ def zip_share(share):
set_rights(zip_path)
return zip_path
def zip_clean():
""" delete zip files older than 1 hour """
if not os.path.exists(app.config['ZIP_FOLDER']):
@@ -708,6 +664,7 @@ def zip_clean():
if mtime + 3600 < time.time():
os.remove(os.path.join(app.config['ZIP_FOLDER'],file))
if __name__ == "__main__":
app.run(debug=True)

0
code/requirements.txt → code/docker-requirements.txt
View File

83
utils/flees-manager.py → code/flees-manager.py
View File

@@ -2,30 +2,9 @@
import argparse,json,sys,os
from shutil import copyfile
from tabulate import tabulate
import hashlib
from datetime import datetime
def get_folder_size(path):
total_size = 0
for dirpath, dirnames, filenames in os.walk(path):
for f in filenames:
fp = os.path.join(dirpath, f)
total_size += os.path.getsize(fp)
return total_size
def get_or_no(key,d,no):
if key in d:
return d[key]
return no
def get_direct_token(share, filename):
if not 'pass_hash' in share:
return None
return hashlib.sha1(
share['pass_hash'].encode('utf-8') + filename.encode('utf-8')
).hexdigest()
from utils.utils import *
from utils.crypt import *
def get_root_path(opts):
root_folder = os.path.dirname(
@@ -37,19 +16,12 @@ def get_root_path(opts):
)
return root_folder
def file_size_human(num):
for x in ['B','KB','MB','GB','TB']:
if num < 1024.0:
if x=='B':
return "%d %s" % (num, x)
return "%3.1f %s" % (num, x)
num /= 1024.0
def list_shares(shares,opts):
table = []
table.append(('Name', 'Path','Public','Password','PassHash','Upload','Overwrite','Direct','Expire','Recipient','Description'))
for share in shares:
public = get_or_no('public',share, False)
public = get_or_none('public',share, False)
passhash = '-'
password = 'pass_hash' in share
if opts.show_password:
@@ -61,11 +33,11 @@ def list_shares(shares,opts):
passhash = share['pass_hash']
else:
passhash = "-"
upload = get_or_no('upload',share, False)
overwrite = get_or_no('overwrite',share, True)
direct = get_or_no('direct_links',share, False) if password else False
expire = get_or_no('expire',share, "-")
description = get_or_no('description',share, "")[0:20]
upload = get_or_none('upload',share, False)
overwrite = get_or_none('overwrite',share, True)
direct = get_or_none('direct_links',share, False) if password else False
expire = get_or_none('expire',share, "-")
description = get_or_none('description',share, "")[0:20]
table.append((
share['name'],
share['path']+"/",
@@ -76,7 +48,7 @@ def list_shares(shares,opts):
overwrite,
direct,
expire,
get_or_no('recipient', share, "")[0:20],
get_or_none('recipient', share, "")[0:20],
description
))
print(tabulate(table, headers = "firstrow"))
@@ -104,7 +76,7 @@ def list_folders(shares,config):
break
(size_num, size_unit) = file_size_human(get_folder_size(
full_path
)).split(" ")
)).split("&nbsp;")
table.append((
folder,
share_name,
@@ -129,7 +101,7 @@ def add_share(shares, config, opts):
if opts.password:
if opts.plain:
share['pass_plain'] = opts.password
share['pass_hash'] = hashlib.sha1(opts.password).hexdigest()
share['pass_hash'] = password_hash(opts.password)
if opts.expire:
try:
date_object = datetime.strptime(opts.expire,"%Y-%m-%d %H:%M")
@@ -193,7 +165,7 @@ def modify_share(shares, config, opts):
# ADD/Change a password
if opts.plain:
share['pass_plain'] = opts.password
share['pass_hash'] = hashlib.sha1(opts.password).hexdigest()
share['pass_hash'] = password_hash(opts.password)
if opts.expire:
if opts.expire == "":
@@ -263,7 +235,6 @@ def print_rest_api(shares, config, opts):
if 'public_url' not in config:
print("Set public_url variable in your config.json")
sys.exit(1)
shares = [share for share in shares if share['name'] == opts.name]
if len(shares) == 0:
print("No such share %s"%( opts.name, ))
@@ -271,7 +242,7 @@ def print_rest_api(shares, config, opts):
share = shares[0]
if opts.type == "list":
print("Link to list contents of the share:")
print("Link to enter the share:")
print("%s/list/%s"%(
config['public_url'],
share['name']
@@ -280,12 +251,14 @@ def print_rest_api(shares, config, opts):
if not 'pass_hash' in share:
print("REST API enabled only if pass_hash is set for share")
sys.exit(1)
crypter = Crypto(config['app_secret_key'])
crypted = crypter.encrypt(share['pass_hash'])
if opts.type == "login":
print("Link to automatically login in the share:")
print("%s/list/%s/%s"%(
config['public_url'],
share['name'],
share['pass_hash']
crypted
))
elif opts.type == "upload":
if 'upload' not in share or not share['upload']:
@@ -296,12 +269,19 @@ def print_rest_api(shares, config, opts):
print("\n# curl -F file=@'the_file_name.ext' %s/upload/%s/%s"%(
config['public_url'],
share['name'],
share['pass_hash']
crypted
))
print("or \n\n# curl -s %s/script/upload/%s/%s | bash /dev/stdin file_to_upload.ext [second.file.ext]"%(
print("\nLink to upload multiple files to the share:")
print("\n# curl -s %s/script/upload/%s/%s | bash /dev/stdin file_to_upload.ext [second.file.ext]"%(
config['public_url'],
share['name'],
share['pass_hash']
crypted
))
print("\nLink to upload multiple files to the share, splitting large files (default 512Mb):")
print("\n# curl -s %s/script/upload_split/%s/%s | bash /dev/stdin [-s split_size_in_Mb] file_to_upload.ext [second.file.ext]"%(
config['public_url'],
share['name'],
crypted
))
elif opts.type == "download":
print("Links to download files:")
@@ -321,13 +301,13 @@ def print_rest_api(shares, config, opts):
print("%s/download/%s/%s/%s"%(
config['public_url'],
share['name'],
share['pass_hash'],
crypted,
filename
))
print("or \n\n# curl -s %s/script/download/%s/%s | bash /dev/stdin [-f]"%(
config['public_url'],
share['name'],
share['pass_hash']
crypted
))
elif opts.type == "direct":
if 'direct_links' not in share or not share['direct_links']:
@@ -356,18 +336,17 @@ def print_rest_api(shares, config, opts):
print("or \n\n# curl -s %s/script/direct/%s/%s | bash /dev/stdin [-f]"%(
config['public_url'],
share['name'],
share['pass_hash']
crypted
))
elif opts.type == "zip":
print("ZIP download:")
print("%s/zip/%s/%s"%(
config['public_url'],
share['name'],
share['pass_hash']
crypted
))
def parse_options():
config_default = os.path.realpath(
os.path.join(
@@ -390,7 +369,7 @@ def parse_options():
## list shares
parser_list = subparsers.add_parser('list', help = "List shares")
parser_list.add_argument('-P', action="store_true", dest="show_password", default = False,
help = "Display hashed passwords")
help = "Display passwords")
## list folders
parser_folders = subparsers.add_parser('folders', help = "List folders and share names")
## Remove

0
utils/requirements.txt → code/manager-requirements.txt
View File

10
code/templates/list.html
View File

@@ -31,8 +31,12 @@
{% else %}
<li>never expires
{% endif %}
{% if overwrite == false %}
<li><span title="Uploaded files need to be uniquely named">overwriting is disabled</span>
{% if upload %}
{% if overwrite %}
<li><span title="Uploaded files with existing names are overwritten">uploads overwrite</span>
{% else %}
<li><span title="Uploaded files with existing names are versioned">uploads versioned</span>
{% endif %}
{% endif %}
<li><a href="{{ url_for('download_zip',name=name) }}" title="Download all the files as one ZIP file. Total size of files must be less than {{ g.max_zip_size }} Mb">Download as zip</a>
<li><a href="{{ url_for('logout',name=name) }}">Logout</a>
@@ -54,7 +58,7 @@
<tr>
<td>
{% if direct %}
<a href="{{ url_for('download_direct', name = name, password = entry.token, filename = entry.name ) }}" title="Direct share link" class=direct>&#x2756;</a>
<a href="{{ url_for('download_direct', name = name, token = entry.token, filename = entry.name ) }}" title="Direct share link" class=direct>&#x2756;</a>
{% endif %}
<a href="{{ url_for('download_file', name = name, filename = entry.name) }}">{{ entry.name }}</a>
<td class=td_right>{{ entry.size|safe }}

1
code/utils/__init__.py Normal file
View File

@@ -0,0 +1 @@

58
code/utils/crypt.py Normal file
View File

@@ -0,0 +1,58 @@
import base64
from Crypto.Cipher import AES
import hashlib
class Crypto:
def __init__(self, secret):
self.secret = add_pad(secret[0:16])
self.cipher = AES.new(self.secret, AES.MODE_ECB)
def encrypt(self, msg):
return base64.urlsafe_b64encode(
self.cipher.encrypt(
add_pad(
msg
)
)
).decode("utf-8")
def decrypt(self, enc):
return remove_pad(
self.cipher.decrypt(
base64.urlsafe_b64decode(
enc
)
).decode("utf-8")
)
def add_pad(string):
""" Add spaces until length is multiple of 16 """
while len(string)%16 != 0:
string+=" "
return string
def get_direct_token(share, filename):
if not 'pass_hash' in share:
return None
return password_hash(
share['pass_hash'] + filename
)
def password_hash(string):
if type(string) == str:
string = string.encode("utf-8")
return hashlib.sha1(
string
).hexdigest()
def remove_pad(string):
""" Remove spaces from right """
return string.rstrip(" ")

49
code/utils/utils.py Normal file
View File

@@ -0,0 +1,49 @@
import os
import hashlib
from datetime import datetime
from flask import current_app as app
def file_date_human(num):
return datetime.fromtimestamp(
num
).strftime(app.config['DATE_FORMAT'])
def file_stat(filename):
s = os.stat(filename)
return {
'size': file_size_MB(s.st_size),
'mtime': file_date_human(s.st_mtime),
'name': os.path.basename(filename)
}
def file_size_human(num,HTML=True):
space = '&nbsp;' if HTML else ' '
for x in [space + 'B', 'KB', 'MB', 'GB', 'TB']:
if num < 1024.0:
if x == space + 'B':
return "%d%s%s" % (num, space, x)
return "%3.1f%s%s" % (num, space, x)
num /= 1024.0
def file_size_MB(num):
return "{:,.2f}".format(num/(1024*1024))
def get_folder_size(path):
total_size = 0
for dirpath, dirnames, filenames in os.walk(path):
for f in filenames:
fp = os.path.join(dirpath, f)
total_size += os.path.getsize(fp)
return total_size
def get_or_none(key,d,none = None):
if key in d:
return d[key]
else:
return none