From 654891a61eeacd963587ef4c2073ae2cbcb91dd3 Mon Sep 17 00:00:00 2001 From: Ville Rantanen Date: Sun, 20 Aug 2023 10:44:16 +0300 Subject: [PATCH] upload with PUT allows streaming --- code/app.py | 79 ++++++++++++++++++++++++++++----------------- code/entrypoint.sh | 3 +- code/templates/mfl | 11 +++---- docker-compose.yaml | 3 +- env.example | 1 + test/run-tests.sh | 73 +++++++++++++++++++++++++++++------------ 6 files changed, 110 insertions(+), 60 deletions(-) diff --git a/code/app.py b/code/app.py index 1635a08..1cf7b7d 100644 --- a/code/app.py +++ b/code/app.py @@ -27,7 +27,7 @@ from utils.files import ( db_maintenance, ) -__VERSION__ = "20230819.0" +__VERSION__ = "20230820.0" app = Flask(__name__) app.config.from_object(__name__) app.config.from_prefixed_env() @@ -42,7 +42,7 @@ def index(): return "", 200 -@app.route("/upload", methods=["POST"]) +@app.route("/upload", methods=["PUT","POST"]) def upload(): """ Upload a file, example CURL: @@ -58,45 +58,64 @@ def upload(): - Additionally, "Expires-Hours" can be used. - Max-Dowloads: -1 means no upper limit + IF using GET, you can upload larger files with pipes + + cat largefile | \ + curl -fL -w "\n" --upload-file - \ + -H "Name: my.file.ext" \ + -H "Max-Downloads: 4000" \ + -H "Expires-Days: 14" \ + -H "Secret: dff789f0bbe8183d32542" \ + "$FLASK_PUBLIC_URL"/upload Returns the file download URL """ + name = request.headers.get("Name", None) + if name is None: + return "Name required", 500 + safe_filename = secure_filename(name) + secret = request.headers.get("Secret", "") + if secret != app.config["ACCESS_TOKEN"]: + return "Error", 401 + max_dl = request.headers.get("Max-Downloads", app.config["DEFAULT_MAX_DL"]) + expires = int(time.time()) + int(app.config["DEFAULT_EXPIRE"]) + if "Expires-days" in request.headers: + expires = int(time.time()) + 24 * 3600 * int( + request.headers.get("Expires-days") + ) + if "Expires-hours" in request.headers: + expires = int(time.time()) + 3600 * int( + request.headers.get("Expires-hours") + ) + + while True: + token = random_token() + folder = os.path.join(app.config["DATAFOLDER"], token) + if not os.path.exists(folder): + break + filename = file_full_path(token, safe_filename) + os.mkdir(folder) + if request.method == "POST": file = request.files.get("file") - name = request.headers.get("Name", None) - if name is None: - return "Name required", 500 - secret = request.headers.get("Secret", "") - if secret != app.config["ACCESS_TOKEN"]: - return "Error", 401 - max_dl = request.headers.get("Max-Downloads", app.config["DEFAULT_MAX_DL"]) - expires = int(time.time()) + int(app.config["DEFAULT_EXPIRE"]) - if "Expires-days" in request.headers: - expires = int(time.time()) + 24 * 3600 * int( - request.headers.get("Expires-days") - ) - if "Expires-hours" in request.headers: - expires = int(time.time()) + 3600 * int( - request.headers.get("Expires-hours") - ) - if file: - safe_filename = secure_filename(name) - while True: - token = random_token() - folder = os.path.join(app.config["DATAFOLDER"], token) - if not os.path.exists(folder): - break - os.mkdir(folder) - filename = file_full_path(token, safe_filename) file.save(filename) - db_store_file(token, safe_filename, expires, max_dl) - download_url = file_full_url(token, safe_filename) - return "File uploaded\n%s\n" % (download_url,), 200 else: return "Use the 'file' variable to upload\n", 400 + if request.method == "PUT": + chunk_size = 1024 * 1024 * 64 # 64Mb + with open(filename, 'wb') as f: + while True: + chunk = request.stream.read(chunk_size) + if not chunk: + break + f.write(chunk) + + db_store_file(token, safe_filename, expires, max_dl) + download_url = file_full_url(token, safe_filename) + return "File uploaded\n%s\n" % (download_url,), 200 @app.route("/details//", methods=["GET"]) def details(token, name): diff --git a/code/entrypoint.sh b/code/entrypoint.sh index ce54cb7..4fd0398 100644 --- a/code/entrypoint.sh +++ b/code/entrypoint.sh @@ -8,6 +8,7 @@ export FLASK_CONF="/data/config.json" export SERVER=gunicorn export PID="flees.pid" export WORKERS +export TIMEOUT if [[ $( stat -c %u /data ) -ne $( id -u ) ]]; then echo User id and /data folder owner do not match @@ -19,4 +20,4 @@ set -eu . /opt/venv/bin/activate sh ./init_db.sh "$FLASK_DB" -exec "$SERVER" -w $WORKERS 'app:app' --pid="$PID" -b 0.0.0.0:5000 +exec "$SERVER" -w $WORKERS --timeout $TIMEOUT 'app:app' --pid="$PID" -b 0.0.0.0:5000 diff --git a/code/templates/mfl b/code/templates/mfl index bc57ee7..3d63498 100755 --- a/code/templates/mfl +++ b/code/templates/mfl @@ -104,8 +104,7 @@ _write() { _write_folder() { # name, file tar c "$2" | \ - curl -fL -w "\n" -F file="@-" -X POST \ - --progress-bar \ + curl -fL -w "\n" -g --upload-file - \ -H "Name: $1" \ -H "Max-Downloads: $MAXDL" \ -H "Expires-Days: $MAXDAYS" \ @@ -113,8 +112,7 @@ _write_folder() { # name, file "$MFL_ROOTURL"/upload | cat } _write_file() { # name, file - curl -fL -w "\n" -F file="@$2" -X POST \ - --progress-bar \ + curl -fL -w "\n" -g --upload-file "$2" \ -H "Name: $1" \ -H "Max-Downloads: $MAXDL" \ -H "Expires-Days: $MAXDAYS" \ @@ -123,8 +121,7 @@ _write_file() { # name, file } _write_stdin() { # name cat - | \ - curl -fL -w "\n" -F file="@-" -X POST \ - --progress-bar \ + curl -fL -w "\n" -g --upload-file - \ -H "Name: $1" \ -H "Max-Downloads: $MAXDL" \ -H "Expires-Days: $MAXDAYS" \ @@ -211,7 +208,7 @@ for (( i=2; i<=$#; i++ )); do done if [[ -z "$NAME" ]]; then if [[ -n "$FILE" ]]; then - NAME="$( basename ${FILE} )" + NAME="$( basename "${FILE}" )" fi fi diff --git a/docker-compose.yaml b/docker-compose.yaml index adb3001..468bf45 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,7 +15,8 @@ services: - ./data/:/data/ restart: unless-stopped environment: - WORKERS: ${WORKERS} + WORKERS: + TIMEOUT: FLASK_APP_SECRET_KEY: FLASK_ACCESS_TOKEN: FLASK_PUBLIC_URL: diff --git a/env.example b/env.example index 823234f..d88db46 100644 --- a/env.example +++ b/env.example @@ -3,6 +3,7 @@ UID=1000 GID=1000 TZ=Europe/Helsinki WORKERS=4 +TIMEOUT=1800 FLASK_APP_SECRET_KEY=8a36bfea77d842386a2a0c7c3e044228363dfddadc01fade4b1b78859ffc615b FLASK_ACCESS_TOKEN=dff789f0bbe8183d3254258b33a147d580c1131f39a698c56d3f640ac8415714 FLASK_PUBLIC_URL=http://localhost:8136 diff --git a/test/run-tests.sh b/test/run-tests.sh index 8680445..00d9c44 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -103,7 +103,7 @@ test -f "$IMAGE" || convert -size 640x480 xc:gray $IMAGE function t00-rebuild-docker() { CWD=$PWD cd .. - docker-compose up --build -d --force-recreate + docker-compose up --build -d --force-recreate -t 0 cd "$CWD" } @@ -133,7 +133,7 @@ function t03-upload-small-file() { } -function t04-upload-large-file() { +function t04-upload-large-file-POST() { pv "$BIG" | \ curl -fL -w "\n" -F file="@-" -X POST \ -H "Name: $BIG" \ @@ -143,6 +143,16 @@ function t04-upload-large-file() { "$FLASK_PUBLIC_URL"/upload } +function t04-upload-large-file-GET() { + pv "$BIG" | \ + curl -fL -w "\n" --upload-file - \ + -H "Name: $BIG" \ + -H "Max-Downloads: 4" \ + -H "Expires-Days: 1" \ + -H "Secret: $FLASK_ACCESS_TOKEN" \ + "$FLASK_PUBLIC_URL"/upload +} + function t05-check-db-manually() { sqlite3 ../data/flees.db "select * FROM files" } @@ -233,22 +243,25 @@ function t10-maintenance-post() { } function t11-get-mfl() { - curl -fL \ + curl -fL \ -H "Secret: $FLASK_ACCESS_TOKEN" \ "$FLASK_PUBLIC_URL"/script/mfl > mfl - chmod +x mfl + chmod +x mfl } function t12-mfl-update() { - ./mfl update + ./mfl update } function t13-mfl-list() { - ./mfl list - ./mfl + ./mfl list + ./mfl } function t14-mfl-upload() { - ./mfl w mfl - ./mfl w -d 1 -m 1 mfl + ./mfl w mfl + ./mfl w -d 1 -m 1 mfl + cat mfl | ./mfl w mfl + ./mfl w . "folder with spaces" + ./mfl w "$SMALL" } @@ -279,19 +292,37 @@ _getchoice() { } -next_task=0 -while true; do - choice=$( _getchoice $next_task ) - if [[ -z "$choice" ]]; then break; fi - _title "$choice" - set -x - "$choice" - set +x - _title "" - #~ next_task=$(( next_task + 1 )) - next_task=$( _getnext "$choice" ) +if [[ -z "$1" ]]; then -done + next_task=0 + while true; do + choice=$( _getchoice $next_task ) + if [[ -z "$choice" ]]; then break; fi + _title "$choice" + set -x + "$choice" + set +x + _title "" + #~ next_task=$(( next_task + 1 )) + next_task=$( _getnext "$choice" ) + done +else + # user passed commands from cmd line + if [[ "$1" = "all" ]]; then + args=$( _getlist ) + else + args="$@" + fi + + for choice in $args; do + _title "$choice" + set -x + "$choice" + set +x + _title "" + if [[ "$choice" = "t00-rebuild-docker" ]]; then sleep 3; fi + done +fi