From 4eab9a34cd963db239226fdb7a2b83f25be74567 Mon Sep 17 00:00:00 2001 From: Q Date: Sun, 14 Jan 2024 09:25:24 +0200 Subject: [PATCH] hidden files, allow ips --- code/app.py | 85 +++++++++++++++++++++++++++++---------------- code/init_db.sh | 4 ++- code/templates/mfl | 31 ++++++----------- code/utils/files.py | 39 ++++++++++++--------- code/utils/misc.py | 12 ++++++- test/run-tests.sh | 53 ++++++++++++++++++++++++++++ 6 files changed, 155 insertions(+), 69 deletions(-) diff --git a/code/app.py b/code/app.py index 04dd2ec..30977d6 100644 --- a/code/app.py +++ b/code/app.py @@ -1,44 +1,51 @@ # -*- coding: utf-8 -*- +import logging import os import sys import time + from flask import ( Flask, - render_template, jsonify, - request, - url_for, redirect, + render_template, + request, send_from_directory, session, + url_for, ) -from werkzeug.utils import secure_filename from revprox import ReverseProxied -from utils.misc import random_token, hash_password, verify_password, file_date_human from utils.files import ( + db_add_download, + db_delete_file, + db_get_file, + db_maintenance, db_store_file, file_details, - file_list, - file_list_simple, file_full_path, file_full_url, - db_add_download, - db_get_file, - db_delete_file, - db_maintenance, - validate_upload_token, + file_list, + file_list_simple, invalidate_upload_token, new_upload_token, + validate_upload_token, ) -import logging +from utils.misc import ( + file_date_human, + hash_password, + is_ip_allowed, + random_token, + verify_password, +) +from werkzeug.utils import secure_filename logging.basicConfig( level=logging.INFO, format=f"[%(asctime)s] [%(levelname)s] %(message)s", ) -__VERSION__ = "20230828.0" +__VERSION__ = "20240114.0" app = Flask(__name__) app.config.from_object(__name__) app.config.from_prefixed_env() @@ -81,6 +88,8 @@ def upload(): -H "Max-Downloads: 4000" \ -H "Expires-Days: 14" \ -H "Password: mypass" \ + -H "Hidden: true" \ + -H "Allowed-IP: 10.0.0.1,10.0.0.2,10.1.*" \ -H "Secret: dff789f0bbe8183d32542" \ "$FLASK_PUBLIC_URL"/upload @@ -115,15 +124,21 @@ def upload(): max_dl = request.headers.get("Max-Downloads", app.config["DEFAULT_MAX_DL"]) expires = int(time.time()) + int(app.config["DEFAULT_EXPIRE"]) if "Expires-days" in request.headers: - expires = int(time.time()) + 24 * 3600 * int( - request.headers.get("Expires-days") - ) + expires = int(time.time()) + 24 * 3600 * int(request.headers.get("Expires-days")) if "Expires-hours" in request.headers: expires = int(time.time()) + 3600 * int(request.headers.get("Expires-hours")) password = None if "Password" in request.headers: if request.headers["Password"] != "": password = hash_password(request.headers["Password"]) + hidden = None + if "Hidden" in request.headers: + hidden = request.headers["Hidden"].lower() == "true" + allowed_ip = None + if "Allowed-IP" in request.headers: + allowed_ip = request.headers["Allowed-IP"].lower() + if not set(allowed_ip) <= set("1234567890.*, "): + return "IP list contains unknown characters" while True: token = random_token() @@ -150,11 +165,9 @@ def upload(): break f.write(chunk) - db_store_file(token, safe_filename, expires, max_dl, password) + db_store_file(token, safe_filename, expires, max_dl, password, ips=allowed_ip, hidden=hidden) download_url = file_full_url(token, safe_filename) - app.logger.info( - f"Upload: {download_url} MaxDL:{max_dl} Exp:{file_date_human(expires)}" - ) + app.logger.info(f"Upload: {download_url} MaxDL:{max_dl} Exp:{file_date_human(expires)}") if upload_token: invalidate_upload_token(upload_token) return "File uploaded\n%s\n" % (download_url,), 200 @@ -176,9 +189,7 @@ def upload_token(): return "Error", 401 expires = int(time.time()) + int(app.config["DEFAULT_EXPIRE"]) if "Expires-days" in request.headers: - expires = int(time.time()) + 24 * 3600 * int( - request.headers.get("Expires-days") - ) + expires = int(time.time()) + 24 * 3600 * int(request.headers.get("Expires-days")) token = new_upload_token(expires) return token, 200 @@ -197,6 +208,11 @@ def details(token, name): if secret != app.config["ACCESS_TOKEN"]: return "Error", 401 details = file_details(token, name) + if details["allowed_ip"] is not None: + if not is_ip_allowed( + details["allowed_ip"], request.environ.get("HTTP_X_FORWARDED_FOR", request.remote_addr), app + ): + return "Not Allowed", 403 return jsonify(details), 200 @@ -208,6 +224,13 @@ def delete_file(name, token): secret = request.headers.get("Secret", "") if secret != app.config["ACCESS_TOKEN"]: return "Error", 401 + details = file_details(token, name) + if details["allowed_ip"] is not None: + if not is_ip_allowed( + details["allowed_ip"], request.environ.get("HTTP_X_FORWARDED_FOR", request.remote_addr), app + ): + return "Not Allowed", 403 + try: os.remove(os.path.join(app.config["DATAFOLDER"], token, name)) except Exception: @@ -307,13 +330,17 @@ def download_file(token, name): return "Error", 404 db_stat = db_get_file(token, name) if db_stat: - added, expires, downloads, max_dl, password_hash = db_stat + added, expires, downloads, max_dl, password_hash, hidden, allowed_ip = db_stat else: return "Error", 404 + if allowed_ip is not None: + if not is_ip_allowed(allowed_ip, request.environ.get("HTTP_X_FORWARDED_FOR", request.remote_addr), app): + return "Not Allowed", 403 + # check ip list! if downloads >= max_dl and max_dl > -1: - return "Expired", 401 + return "Expired", 403 if expires < time.time(): - return "Expired", 401 + return "Expired", 403 if password_hash: if verify_password(session.get(token, ""), password_hash): pass @@ -324,9 +351,7 @@ def download_file(token, name): return redirect(url_for("login")) db_add_download(token, name) - return send_from_directory( - directory=os.path.join(app.config["DATAFOLDER"], token), path=name - ) + return send_from_directory(directory=os.path.join(app.config["DATAFOLDER"], token), path=name) def print_debug(s): diff --git a/code/init_db.sh b/code/init_db.sh index 467839d..c204473 100644 --- a/code/init_db.sh +++ b/code/init_db.sh @@ -9,7 +9,9 @@ CREATE TABLE IF NOT EXISTS files ( expires integer NOT NULL, downloads integer NOT NULL, max_downloads integer NOT NULL, - passhash text + passhash text, + allowed_ip text, + hidden boolean ); EOF diff --git a/code/templates/mfl b/code/templates/mfl index 5b89321..6596ef5 100755 --- a/code/templates/mfl +++ b/code/templates/mfl @@ -17,6 +17,8 @@ _help() { -d max days to keep (default 30) -m max downloads to keep (default 9999, -1 to disable) -p password + --hidden hidden file (from listings) + --allow comma separated list of ip addresses, accepts * Filename: When writing to share: @@ -88,7 +90,7 @@ _write() { [[ -d "$FILE" ]] && { [[ "${NAME,,}" = *".tar" ]] || NAME="${NAME}".tar - _write_folder "$NAME" "$FILE" + tar c "$2" | _write_stdin "$NAME" return $? } @@ -96,28 +98,9 @@ _write() { _msg "No such file" exit 1 } - _write_file "$NAME" "$FILE" + cat "$FILE" | _write_stdin "$NAME" } -_write_folder() { # name, file - tar c "$2" | \ - curl -fL -w "\n" -g --upload-file - \ - -H "Name: $1" \ - -H "Max-Downloads: $MAXDL" \ - -H "Expires-Days: $MAXDAYS" \ - -H "Secret: $MFL_TOKEN" \ - -H "Password: $PASSWORD" \ - "$MFL_ROOTURL"/upload | cat -} -_write_file() { # name, file - curl -fL -w "\n" -g --upload-file "$2" \ - -H "Name: $1" \ - -H "Max-Downloads: $MAXDL" \ - -H "Expires-Days: $MAXDAYS" \ - -H "Secret: $MFL_TOKEN" \ - -H "Password: $PASSWORD" \ - "$MFL_ROOTURL"/upload | cat -} _write_stdin() { # name cat - | \ curl -fL -w "\n" -g --upload-file - \ @@ -125,7 +108,9 @@ _write_stdin() { # name -H "Max-Downloads: $MAXDL" \ -H "Expires-Days: $MAXDAYS" \ -H "Secret: $MFL_TOKEN" \ + -H "Hidden: $HIDDEN" \ -H "Password: $PASSWORD" \ + -H "Allowed-IP: $ALLOWED" \ "$MFL_ROOTURL"/upload | cat } @@ -218,6 +203,8 @@ done MAXDL=9999 MAXDAYS=30 +HIDDEN=false +ALLOWED="" CMD=list # if stdin comes from stream, default to write [ -t 0 ] || CMD=help @@ -226,6 +213,8 @@ for (( i=2; i<=$#; i++ )); do [[ "${!i}" = "-m" ]] && { MAXDL=${!j}; i=$j; continue; } [[ "${!i}" = "-d" ]] && { MAXDAYS=${!j}; i=$j; continue; } [[ "${!i}" = "-p" ]] && { PASSWORD=${!j}; i=$j; continue; } + [[ "${!i}" = "--allow" ]] && { ALLOWED="${!j}"; i=$j; continue; } + [[ "${!i}" = "--hidden" ]] && { HIDDEN=true; continue; } if [[ -z "$FILE" ]]; then FILE="${!i}" continue diff --git a/code/utils/files.py b/code/utils/files.py index 48d2e2b..bc8ab8a 100644 --- a/code/utils/files.py +++ b/code/utils/files.py @@ -1,10 +1,12 @@ import os -from datetime import datetime -from flask import current_app as app -import time -import sqlite3 import shutil -from .misc import file_size_human, file_date_human, file_time_human, random_token +import sqlite3 +import time +from datetime import datetime + +from flask import current_app as app + +from .misc import file_date_human, file_size_human, file_time_human, random_token def get_db(): @@ -13,14 +15,14 @@ def get_db(): return db, c -def db_store_file(token, name, expires, max_dl, password): +def db_store_file(token, name, expires, max_dl, password, ips=None, hidden=None): db, c = get_db() c.execute( """ - insert into files(token,name,added,expires,downloads,max_downloads,passhash) - values (?,?,?,?,?,?,?) + insert into files(token,name,added,expires,downloads,max_downloads,passhash,allowed_ip,hidden) + values (?,?,?,?,?,?,?,?,?) """, - (token, name, int(time.time()), expires, 0, max_dl, password), + (token, name, int(time.time()), expires, 0, max_dl, password, ips, hidden), ) if c.rowcount > 0: db.commit() @@ -31,7 +33,7 @@ def db_get_file(token, name): db, c = get_db() return db.execute( """ - SELECT added,expires,downloads,max_downloads,passhash + SELECT added,expires,downloads,max_downloads,passhash,hidden,allowed_ip FROM files WHERE token = ? AND name = ? """, (token, name), @@ -42,9 +44,11 @@ def db_get_files(): db, c = get_db() return db.execute( """ - SELECT token,name,added,expires,downloads,max_downloads,passhash + SELECT token,name,added,expires,downloads,max_downloads,passhash,allowed_ip FROM files - WHERE expires > ? and (downloads < max_downloads or max_downloads = -1) + WHERE expires > ? + AND (downloads < max_downloads or max_downloads = -1) + AND (hidden IS NULL OR hidden = 0) ORDER BY added """, (time.time(),), @@ -168,8 +172,7 @@ def file_age(path): diff = now - then return ( diff, - "%03d d %s" - % (diff.days, datetime.utcfromtimestamp(diff.seconds).strftime("%H:%M:%S")), + "%03d d %s" % (diff.days, datetime.utcfromtimestamp(diff.seconds).strftime("%H:%M:%S")), ) @@ -179,7 +182,7 @@ def file_details(token, name): s = os.stat(full_path) db_stat = db_get_file(token, name) if db_stat: - added, expires, downloads, max_dl, password = db_stat + added, expires, downloads, max_dl, password, hidden, allowed_ip = db_stat else: return {} return { @@ -192,6 +195,8 @@ def file_details(token, name): "downloaded": downloads, "max-dl": max_dl, "protected": password is not None, + "hidden": hidden, + "allowed_ip": allowed_ip, } except FileNotFoundError: return {} @@ -214,7 +219,9 @@ def file_list(): added = file_date_human(file[2]) expiry = file_date_human(file[3]) pw = " (PW)" if file[6] else "" - details.append(f"{added}/{expiry} {file[4]:4d}/{file[5]:4d} {url}{pw}") + ips = f" [{file[7]}]" if file[7] else "" + + details.append(f"{added}/{expiry} {file[4]:4d}/{file[5]:4d} {url}{pw}{ips}") return details diff --git a/code/utils/misc.py b/code/utils/misc.py index 66c23af..cec3f1b 100644 --- a/code/utils/misc.py +++ b/code/utils/misc.py @@ -1,6 +1,8 @@ -from datetime import datetime +import fnmatch import secrets import string +from datetime import datetime + import passlib.hash VALID_TOKEN_CHARS = string.digits + string.ascii_letters @@ -38,3 +40,11 @@ def hash_password(password): def verify_password(password, hash): return passlib.hash.argon2.verify(password, hash) + + +def is_ip_allowed(allowed_ip, ip, app): + for rule in allowed_ip.split(","): + rule = rule.strip() + if fnmatch.fnmatch(ip, rule): + return True + return False diff --git a/test/run-tests.sh b/test/run-tests.sh index cf94f46..f3bfdd0 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -87,6 +87,12 @@ _title() { set -e +if [[ ! -d ../test ]]; then + echo run in the test folder + exit 1 +fi + + BIG="big file(1).ext" BIGS="big_file1.ext" SMALL="small file" @@ -316,7 +322,54 @@ function t17-new-upload_token() { } +function t18-upload_hidden() { + url=$( cat "$IMAGE" | \ + curl -fL -w "\n" -F file="@-" -X POST \ + -H "Name: $IMAGE" \ + -H "Secret: $FLASK_ACCESS_TOKEN" \ + -H "Hidden: true" \ + -H "Max-Downloads: 1" \ + "$FLASK_PUBLIC_URL"/upload | grep ^http ) + curl "$url" > /dev/null + + ./mfl w -m 1 --hidden "$IMAGE" + + curl -fL -w "\n" \ + -H "Secret: $FLASK_ACCESS_TOKEN" \ + "$FLASK_PUBLIC_URL"/ls +} + + +function t19-upload_ip_allow() { + url=$( cat "$IMAGE" | \ + curl -fL -w "\n" -F file="@-" -X POST \ + -H "Name: $IMAGE" \ + -H "Secret: $FLASK_ACCESS_TOKEN" \ + -H "Allowed-IP: *" \ + "$FLASK_PUBLIC_URL"/upload | grep ^http ) + curl -s -D /dev/stderr "$url" | head -c 1 | head -c 0 + + url=$( cat "$IMAGE" | \ + curl -fL -w "\n" -F file="@-" -X POST \ + -H "Name: $IMAGE" \ + -H "Secret: $FLASK_ACCESS_TOKEN" \ + -H "Allowed-IP: 10.0.0.10, 172.20.*" \ + "$FLASK_PUBLIC_URL"/upload | grep ^http ) + curl -s -D /dev/stderr "$url" | head -c 1 | head -c 0 + + url=$( cat "$IMAGE" | \ + curl -fL -w "\n" -F file="@-" -X POST \ + -H "Name: $IMAGE" \ + -H "Secret: $FLASK_ACCESS_TOKEN" \ + -H "Allowed-IP: 10.0.0.10, 12.12.12.12" \ + "$FLASK_PUBLIC_URL"/upload | grep ^http ) + curl -s -D /dev/stderr "$url" | head -c 1 | head -c 0 + + ./mfl w -m 1 --allow '*' "$IMAGE" + ./mfl w -m 1 --allow '10.0.0.1, 10.0.0.*' "$IMAGE" + ./mfl list +} _getlist() { declare -F | awk '{ print $3 }' | grep -v ^_