Files
flees/code/app.py
2018-03-10 15:28:34 +02:00

694 lines
20 KiB
Python

# -*- coding: utf-8 -*-
import os,sys,time,stat
import json
from datetime import datetime
from flask import Flask, render_template, jsonify, current_app, Response, \
redirect, url_for, request, g, session, send_file, send_from_directory
from werkzeug.utils import secure_filename
import zipfile
from multiprocessing import Process
from revprox import ReverseProxied
from utils.utils import *
from utils.crypt import *
__FLEES_VERSION__ = "20180310.0"
app = Flask(__name__)
app.config.from_object(__name__)
# Read config from json !
config_values = json.load(open(os.getenv('FLEES_CONFIG'),'rt'))
app.config['SITE_NAME'] = config_values['site_name']
app.config['UPLOAD_FOLDER'] = config_values['data_folder']
app.config['SHARES_FILE'] = config_values['shares_file']
app.config['ZIP_FOLDER'] = config_values['zip_folder']
app.config['MAX_ZIP_SIZE'] = config_values['max_zip_size'] # megabytes
app.config['DATE_FORMAT'] = config_values['date_format']
app.config['UID'] = config_values['uid']
app.config['GID'] = config_values['gid']
app.config['DEBUG'] = config_values['debug']
if 'notifier' in config_values:
if len(config_values['notifier']) > 0:
notifier_config = config_values['notifier'].split(":")
imported = getattr(__import__(
notifier_config[0],
fromlist=[notifier_config[1]]
),
notifier_config[1]
)
app.config['notifier'] = imported()
app.secret_key = config_values['app_secret_key']
app.wsgi_app = ReverseProxied(app.wsgi_app)
@app.before_request
def before_request():
g.shares = json.load(open(app.config['SHARES_FILE'],'rt'))
g.version = __FLEES_VERSION__
g.site_name = app.config['SITE_NAME']
g.max_zip_size = app.config['MAX_ZIP_SIZE']
@app.route("/")
def index():
public_shares = []
for share in g.shares:
public = get_or_none('public', share)
expired = is_expired(share)
authenticated_share = get_share(share['name'])
password_set = False
if authenticated_share[0]:
password_set = authenticated_share[1]['authenticated'] == 'hash'
if not expired:
if public or password_set:
public_shares.append({
'name': share['name'],
'expire': get_or_none('expire', share),
'upload': get_or_none('upload', share),
'password_set': password_set,
'description': get_or_none('description', share, '')
})
return render_template("index.html", entries=public_shares)
@app.route('/authenticate/<name>', methods=['GET','POST'])
def authenticate(name):
if request.method == 'GET':
return render_template('authenticate.html',name=name)
if request.method == 'POST':
user_password = request.form['password'].encode('utf-8')
session[name] = password_hash(user_password, app.secret_key)
return redirect(url_for('list_view',name=name))
@app.route('/upload/<name>/<token>', methods=['POST'])
@app.route('/upload', methods=['POST'])
def upload(name = None, token = None):
if request.method == 'POST':
file = request.files['file']
if name == None:
name = request.form['name']
(ok,share) = get_share(name, token = token)
if not ok:
return share
if not get_or_none('upload', share) == True:
return "Upload not allowed\n",400
if file:
filename = os.path.join(
share['path'],
secure_filename(
file.filename
)
)
if get_or_none('overwrite', share) == False:
if os.path.exists(filename):
file_versionize(filename)
#~ return "Overwrite forbidden", 403
file.save(filename)
set_rights(filename)
notify({
"recipient": get_or_none('recipient', share),
"share": name,
"filename": filename,
"operation": "upload"
})
if 'from_gui' in request.form:
if request.form['from_gui'] == "true":
return redirect(url_for('list_view',name=name))
return "File uploaded\n", 200
else:
return "Use the 'file' variable to upload\n",400
@app.route('/upload_join/<name>/<token>', methods=['POST'])
def upload_join_splitted(name, token):
if request.method == 'POST':
(ok,share) = get_share(name, token = token)
if not ok:
return share
if not get_or_none('upload', share) == True:
return "Upload not allowed",400
if not 'filename' in request.form:
return "No filename given", 400
parts = []
for part in range(100):
filename = os.path.join(
share['path'],
"%s.part.%03d"%(
request.form['filename'],
part
)
)
if os.path.exists(filename):
parts.append(filename)
part_existed = part
if len(parts) == 0:
return "Invalid partial filename\n", 400
if not len(parts) == part_existed + 1:
return "Parts missing\n", 400
target_name = os.path.join(
share['path'],
request.form['filename']
)
if get_or_none('overwrite', share) == False:
if os.path.exists(target_name):
file_versionize(target_name)
try:
begin = uploadJoiner(target_name, parts)
except:
return "Joining failed\n", 400
return "Joining started\n", 200
@app.route('/send/<name>', methods=['GET'])
def send(name):
(ok,share) = get_share(name)
if not ok:
return share
return render_template('send.html',name=name)
@app.route('/files/<name>/<token>', methods=['GET'])
def list_files(name, token):
(ok,share) = get_share(name, token = token)
if not ok:
return share
files = []
for file in iter_folder_files(share['path']):
files.append(path2url(file))
files.append("")
return "\n".join(files), 200
@app.route('/list/<name>/<token>', methods=['GET'])
@app.route('/list/<name>', methods=['GET'])
def list_view(name, token = None):
(ok,share) = get_share(name, token = token)
if not ok:
return share
if token != None and 'pass_hash' in share:
session[name] = share['pass_hash']
return redirect(url_for('list_view',name=name))
files = []
for file in iter_folder_files(share['path']):
fp = os.path.join(share['path'],file)
status = file_stat(fp)
status.update({
'token': get_direct_token(share, file),
'name': file,
'url': path2url(file)
})
files.append(status)
# direct share links not allowed if password isnt set
allow_direct = get_or_none('direct_links', share) if get_or_none('pass_hash', share) else False
upload = get_or_none('upload', share)
overwrite = get_or_none('overwrite', share)
if not upload:
overwrite = False
return render_template(
"list.html",
name = share['name'],
entries = files,
password = get_or_none('pass_hash', share),
public = get_or_none('public', share),
upload = upload,
overwrite = overwrite,
direct = allow_direct,
expire = get_or_none('expire', share),
description = get_or_none('description', share, "")
)
@app.route('/logout/<name>', methods=['GET'])
def logout(name):
if name in session:
del session[name]
return render_template(
"logout.html",
name = name
)
@app.route('/direct/<name>/<token>/<path:filename>', methods=['GET'])
def download_direct(name,token,filename):
(ok,share) = get_share(name, require_auth = False)
if not ok:
return share
allow_direct = get_or_none('direct_links', share)
if allow_direct != True:
return 'Direct download not allowed', 403
if not is_path_safe(filename):
return 'Incorrect relative path'+filename, 403
file_token = get_direct_token(share, filename)
if file_token == None:
return 'Cannot generate token', 400
if file_token != token:
return 'Incorrect token', 403
file_path = os.path.join(share['path'], filename)
if not os.path.exists(file_path):
return 'No such file', 404
notify({
"recipient": get_or_none('recipient', share),
"share": name,
"filename": file_path,
"operation": "direct_download"
})
return send_from_directory(directory=share['path'], filename=filename)
@app.route('/download/gui/<name>/<path:filename>', methods=['GET'])
def download_gui(name, filename):
return download_file(name, filename, token = None)
@app.route('/download/<name>/<token>/<path:filename>', methods=['GET'])
def download_token(name, filename, token):
return download_file(name, filename, token = token)
@app.route('/zip/<name>/<token>', methods=['GET'])
@app.route('/zip/<name>', methods=['GET'])
def download_zip(name, token = None):
(ok,share) = get_share(name, token = token)
if not ok:
return share
folder_size = get_folder_size(share['path'])
if folder_size/(1024*1024) > app.config['MAX_ZIP_SIZE']:
return "Maximum ZIP size exceeded", 400
zip_clean()
zip_path = zip_share(share)
notify({
"recipient": get_or_none('recipient', share),
"share": name,
"filename": name + ".zip",
"operation": "zip_download"
})
return send_file(
zip_path,
as_attachment = True,
attachment_filename = name + ".zip"
)
@app.route('/script/upload/<name>/<token>', methods=['GET'])
def script_upload(name = None, token = None):
(ok,share) = get_share(name, token = token)
if not ok:
return share
if not get_or_none('upload', share) == True:
return "Upload not allowed",400
return """#!/bin/bash
test -n "$1" || {
echo "Add files to upload as argument"
exit 1
}
CAT=$( which cat )
which pv &> /dev/null && CAT=$( which pv )
ROOTURL="%s"
SHARE="%s"
TOKEN="%s"
send_file() {
$CAT "$file_name" | curl -F "file=@-;filename=${base_name}" ${ROOTURL}upload/${SHARE}/${TOKEN}
}
send_folder() {
which pv &> /dev/null && printf -v dusize -- "--size %%dk" $( du -s -k "$file_name" | cut -f1 )
tar cz "$file_name" | $CAT $dusize - | curl -F "file=@-;filename=${base_name}.tgz" ${ROOTURL}upload/${SHARE}/${TOKEN}
}
for file_name in "$@"; do
base_name=$( basename "$file_name" )
test -f "$file_name" && {
printf "Sending file: %%s\n" "$file_name"
send_file
continue
}
test -d "$file_name" && {
printf "Sending folder: %%s\n" "$file_name"
send_folder
continue
}
done
"""%(
request.url_root,
name,
token
)
@app.route('/script/download/<name>/<token>', methods=['GET'])
def script_download(name = None, token = None):
(ok,share) = get_share(name, token = token)
if not ok:
return share
files = []
for file in sorted(os.listdir(share['path'])):
fp = os.path.join(share['path'],file)
if os.path.isdir(fp):
continue
if file.startswith("."):
continue
status = file_stat(fp)
status.update({
'token': get_direct_token(share, file)
})
files.append(status)
script = """#!/bin/bash
test "$1" = "-h" && {
echo "Add argument -f to overwrite files"
exit 0
}
test "$1" = "-f" && FORCE=1
which curl &> /dev/null || {
echo "curl required"
exit 1
}
ROOTURL="%s"
SHARE="%s"
TOKEN="%s"
get_file() {
WRITE=0
FILENAME="$1"
test "$FORCE" = "1" && WRITE=1
test -f "${FILENAME}" || WRITE=1
test "$WRITE" = "1" && {
echo Downloading ${FILENAME}
curl "${ROOTURL}download/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}"
} || {
echo Skipping ${FILENAME}
}
}
"""%(
request.url_root,
name,
token
)
for file in files:
script += 'get_file "%s"\n'%(
file['name'],
)
return script
@app.route('/script/direct/<name>/<token>', methods=['GET'])
def script_direct(name = None, token = None):
(ok,share) = get_share(name, token = token)
if not ok:
return share
files = []
for file in sorted(os.listdir(share['path'])):
fp = os.path.join(share['path'],file)
if os.path.isdir(fp):
continue
if file.startswith("."):
continue
status = file_stat(fp)
status.update({
'token': get_direct_token(share, file)
})
files.append(status)
script = """#!/bin/bash
test "$1" = "-h" && {
echo "Add argument -f to overwrite files"
exit 0
}
test "$1" = "-f" && FORCE=1
which curl &> /dev/null || {
echo "curl required"
exit 1
}
ROOTURL="%s"
SHARE="%s"
get_file() {
WRITE=0
FILENAME="$1"
TOKEN="$2"
test "$FORCE" = "1" && WRITE=1
test -f "${FILENAME}" || WRITE=1
test "$WRITE" = "1" && {
echo Downloading ${FILENAME}
curl "${ROOTURL}direct/${SHARE}/${TOKEN}/${FILENAME}" > "${FILENAME}"
} || {
echo Skipping ${FILENAME}
}
}
"""%(
request.url_root,
name,
)
for file in files:
script += 'get_file "%s" "%s"\n'%(
file['name'],
file['token'],
)
return script
@app.route('/script/upload_split/<name>/<token>', methods=['GET'])
def script_upload_split(name = None, token = None):
(ok,share) = get_share(name, token = token)
if not ok:
return share
if not get_or_none('upload', share) == True:
return "Upload not allowed",400
return """#!/bin/bash
test -n "$1" || {
echo "Usage: [-s SplitMegabytes] file/folder [files/folders]"
exit 1
}
MAXBYTES=$(( 512 * 1024 * 1024 ))
for (( i=1; i<=$#; i++ )); do
j=$(( $i + 1 ))
[[ ${!i} = "-s" ]] && {
MAXBYTES=$(( ${!j} * 1024 * 1024 ))
shift 2
}
done
CAT=$( which cat )
which pv &> /dev/null && CAT=$( which pv )
ROOTURL="%s"
SHARE="%s"
TOKEN="%s"
send_file() {
$CAT "$file_name" | split -d -a 3 -b $MAXBYTES \
--filter="curl -f -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \
- "$base_name.part." && \
curl -F "filename=$base_name" ${ROOTURL}upload_join/${SHARE}/${TOKEN}
}
send_folder() {
which pv &> /dev/null && printf -v dusize -- "--size %%dk" $( du -s -k "$file_name" | cut -f1 )
tar c "$file_name" | $CAT $dusize - | split -d -a 3 -b $MAXBYTES \
--filter="curl -f -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \
- "$base_name.tar.part." && \
curl -F "filename=$base_name.tar" ${ROOTURL}upload_join/${SHARE}/${TOKEN}
}
echo "Splitting to $(( $MAXBYTES / 1024 / 1024 )) Mb chunks"
for file_name in "$@"; do
base_name=$( basename "$file_name" )
test -f "$file_name" && {
printf "Sending file: %%s\n" "$file_name"
send_file
continue
}
test -d "$file_name" && {
printf "Sending folder: %%s\n" "$file_name"
send_folder
continue
}
done
"""%(
request.url_root,
name,
token
)
class uploadJoiner:
def __init__(self, target_name, parts):
self.target_name = target_name
self.parts = parts
self.chunk_size = 10 * 1024 * 1024
p = Process(target=self.run, args=())
p.daemon = True
p.start()
def run(self):
with open(self.target_name,'wb') as writer:
for part in self.parts:
with open(part,'rb') as reader:
for chunk in iter(lambda: reader.read(self.chunk_size), b""):
writer.write(chunk)
set_rights(self.target_name)
for part in self.parts:
os.remove(part)
def download_file(name, filename, token = None):
(ok,share) = get_share(name, token = token)
if not ok:
return share
if not is_path_safe(filename):
return "Incorrect path", 403
file_path = os.path.join(share['path'], filename)
if not os.path.exists(file_path):
return 'No such file, '+file_path, 404
notify({
"recipient": get_or_none('recipient', share),
"share": name,
"filename": file_path,
"operation": "download"
})
return send_from_directory(directory=share['path'], filename=filename)
def file_versionize(filename):
""" Move file to versioned with integer """
stats = file_stat(filename)
basename, extension = os.path.splitext(stats['name'])
version = 1
while True:
new_name = os.path.join(
os.path.dirname(filename),
secure_filename("%s.v%d%s"%(
basename,
version,
extension
))
)
if os.path.exists(new_name):
version += 1
else:
break
os.rename(filename,new_name)
def get_share(name, require_auth = True, token = None):
share = [x for x in g.shares if x['name'] == name]
if len(share) < 1:
return (False,redirect(url_for('authenticate',name=name)))
share = share[0]
if is_expired(share):
return (False, 'Share has expired')
authenticated = "no-pass"
if not token == None:
if has_token(token, share):
require_auth = False
authenticated = "token"
if require_auth:
if 'pass_hash' in share:
authenticated = False
if name in session:
if session[name] == share['pass_hash']:
authenticated = "hash"
if not authenticated:
return (False,redirect(url_for('authenticate',name=name)))
if not 'path' in share:
return (False,'no path defined')
share.update({
"path": os.path.join(
app.config['UPLOAD_FOLDER'],
share['path']
),
"authenticated": authenticated
})
if not os.path.exists(share['path']):
makedirs_rights(share['path'])
return (True,share)
def is_expired(share):
expires = get_or_none('expire', share)
if expires:
if datetime.now() > datetime.strptime(expires, app.config['DATE_FORMAT']):
return True
return False
def print_debug(s):
if app.config['DEBUG']:
sys.stderr.write(str(s)+"\n")
sys.stderr.flush()
def makedirs_rights(path):
# os.makedirs with chown
path_list = path.split(os.sep)
for p in range(len(path_list)):
current_path = os.sep.join(path_list[0:(p+1)])
if not os.path.exists(current_path):
os.mkdir(current_path)
set_rights(current_path)
def notify(msg):
if 'notifier' in app.config:
msg['environment'] = request.environ
app.config['notifier'].notify(msg)
def set_rights(path):
os.chown(path, app.config['UID'], app.config['GID'])
st = os.stat(path)
if app.config['UID'] > 0:
os.chmod(path, st.st_mode | stat.S_IRUSR | stat.S_IWUSR)
if app.config['GID'] > 0:
os.chmod(path, st.st_mode | stat.S_IRGRP | stat.S_IWGRP)
def zip_share(share):
if not os.path.exists(app.config['ZIP_FOLDER']):
os.makedirs(app.config['ZIP_FOLDER'])
set_rights(app.config['ZIP_FOLDER'])
zip_path = os.path.join(
app.config['ZIP_FOLDER'],
"%s-%d.zip"%(
share['name'],
time.time()
)
)
zf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
for file in iter_folder_files(share['path']):
fp = os.path.join(share['path'], file)
if os.path.isdir(fp):
continue
print_debug(fp)
zf.write(
fp,
arcname = os.path.join(share['name'],file)
)
zf.close()
set_rights(zip_path)
return zip_path
def zip_clean():
""" delete zip files older than 1 hour """
if not os.path.exists(app.config['ZIP_FOLDER']):
return
for file in os.listdir(app.config['ZIP_FOLDER']):
if not file.endswith("zip"):
continue
mtime = os.stat(
os.path.join(app.config['ZIP_FOLDER'],file)
).st_mtime
if mtime + 3600 < time.time():
os.remove(os.path.join(app.config['ZIP_FOLDER'],file))
if __name__ == "__main__":
zip_clean()
app.run(debug=True)