upload with PUT allows streaming

code/app.py

@@ -27,7 +27,7 @@ from utils.files import (
     db_maintenance,
 )
 
-__VERSION__ = "20230819.0"
+__VERSION__ = "20230820.0"
 app = Flask(__name__)
 app.config.from_object(__name__)
 app.config.from_prefixed_env()
@@ -42,7 +42,7 @@ def index():
     return "", 200
 
 
-@app.route("/upload", methods=["POST"])
+@app.route("/upload", methods=["PUT","POST"])
 def upload():
     """
     Upload a file, example CURL:
@@ -58,15 +58,23 @@ def upload():
     - Additionally, "Expires-Hours" can be used.
     - Max-Dowloads: -1  means no upper limit
 
+    IF using GET, you can upload larger files with pipes
+
+    cat largefile | \
+        curl -fL -w "\n" --upload-file - \
+        -H "Name: my.file.ext" \
+        -H "Max-Downloads: 4000" \
+        -H "Expires-Days: 14" \
+        -H "Secret: dff789f0bbe8183d32542" \
+        "$FLASK_PUBLIC_URL"/upload
 
     Returns the file download URL
     """
 
-    if request.method == "POST":
-        file = request.files.get("file")
     name = request.headers.get("Name", None)
     if name is None:
         return "Name required", 500
+    safe_filename = secure_filename(name)
     secret = request.headers.get("Secret", "")
     if secret != app.config["ACCESS_TOKEN"]:
         return "Error", 401
@@ -81,22 +89,33 @@ def upload():
             request.headers.get("Expires-hours")
         )
 
-        if file:
-            safe_filename = secure_filename(name)
     while True:
         token = random_token()
         folder = os.path.join(app.config["DATAFOLDER"], token)
         if not os.path.exists(folder):
             break
-            os.mkdir(folder)
     filename = file_full_path(token, safe_filename)
+    os.mkdir(folder)
+
+    if request.method == "POST":
+        file = request.files.get("file")
+        if file:
             file.save(filename)
-            db_store_file(token, safe_filename, expires, max_dl)
-            download_url = file_full_url(token, safe_filename)
-            return "File uploaded\n%s\n" % (download_url,), 200
         else:
             return "Use the 'file' variable to upload\n", 400
 
+    if request.method == "PUT":
+        chunk_size = 1024 * 1024 * 64 # 64Mb
+        with open(filename, 'wb') as f:
+            while True:
+                chunk = request.stream.read(chunk_size)
+                if not chunk:
+                  break
+                f.write(chunk)
+
+    db_store_file(token, safe_filename, expires, max_dl)
+    download_url = file_full_url(token, safe_filename)
+    return "File uploaded\n%s\n" % (download_url,), 200
 
 @app.route("/details/<token>/<name>", methods=["GET"])
 def details(token, name):

code/entrypoint.sh

@@ -8,6 +8,7 @@ export FLASK_CONF="/data/config.json"
 export SERVER=gunicorn
 export PID="flees.pid"
 export WORKERS
+export TIMEOUT
 
 if [[ $( stat -c %u /data ) -ne $( id -u ) ]]; then
   echo User id and /data folder owner do not match
@@ -19,4 +20,4 @@ set -eu
 . /opt/venv/bin/activate
 sh ./init_db.sh "$FLASK_DB"
 
-exec "$SERVER" -w $WORKERS 'app:app' --pid="$PID" -b 0.0.0.0:5000
+exec "$SERVER" -w $WORKERS --timeout $TIMEOUT 'app:app' --pid="$PID" -b 0.0.0.0:5000

code/templates/mfl

@@ -104,8 +104,7 @@ _write() {
 
 _write_folder() { # name, file
   tar c "$2" | \
-  curl -fL -w "\n" -F file="@-" -X POST \
+  curl -fL -w "\n" -g --upload-file - \
-    --progress-bar \
     -H "Name: $1" \
     -H "Max-Downloads: $MAXDL" \
     -H "Expires-Days: $MAXDAYS" \
@@ -113,8 +112,7 @@ _write_folder() { # name, file
     "$MFL_ROOTURL"/upload | cat
 }
 _write_file() { # name, file
-  curl -fL -w "\n" -F file="@$2" -X POST \
+  curl -fL -w "\n" -g --upload-file "$2" \
-    --progress-bar \
     -H "Name: $1" \
     -H "Max-Downloads: $MAXDL" \
     -H "Expires-Days: $MAXDAYS" \
@@ -123,8 +121,7 @@ _write_file() { # name, file
 }
 _write_stdin() { # name
   cat - | \
-  curl -fL -w "\n" -F file="@-" -X POST \
+  curl -fL -w "\n" -g --upload-file - \
-    --progress-bar \
     -H "Name: $1" \
     -H "Max-Downloads: $MAXDL" \
     -H "Expires-Days: $MAXDAYS" \
@@ -211,7 +208,7 @@ for (( i=2; i<=$#; i++ )); do
 done
 if [[ -z "$NAME" ]]; then
   if [[ -n "$FILE" ]]; then
-      NAME="$( basename ${FILE} )"
+      NAME="$( basename "${FILE}" )"
   fi
 fi
 

docker-compose.yaml

@@ -15,7 +15,8 @@ services:
       - ./data/:/data/
     restart: unless-stopped
     environment:
-      WORKERS: ${WORKERS}
+      WORKERS:
+      TIMEOUT:
       FLASK_APP_SECRET_KEY:
       FLASK_ACCESS_TOKEN:
       FLASK_PUBLIC_URL:

env.example

@@ -3,6 +3,7 @@ UID=1000
 GID=1000
 TZ=Europe/Helsinki
 WORKERS=4
+TIMEOUT=1800
 FLASK_APP_SECRET_KEY=8a36bfea77d842386a2a0c7c3e044228363dfddadc01fade4b1b78859ffc615b
 FLASK_ACCESS_TOKEN=dff789f0bbe8183d3254258b33a147d580c1131f39a698c56d3f640ac8415714
 FLASK_PUBLIC_URL=http://localhost:8136

test/run-tests.sh

@@ -103,7 +103,7 @@ test -f "$IMAGE" || convert -size 640x480 xc:gray $IMAGE
 function t00-rebuild-docker() {
   CWD=$PWD
   cd ..
-  docker-compose up --build -d --force-recreate
+  docker-compose up --build -d --force-recreate -t 0
   cd "$CWD"
 }
 
@@ -133,7 +133,7 @@ function t03-upload-small-file() {
 
 }
 
-function t04-upload-large-file() {
+function t04-upload-large-file-POST() {
   pv "$BIG" | \
     curl -fL -w "\n" -F file="@-" -X POST \
     -H "Name: $BIG" \
@@ -143,6 +143,16 @@ function t04-upload-large-file() {
     "$FLASK_PUBLIC_URL"/upload
 }
 
+function t04-upload-large-file-GET() {
+  pv "$BIG" | \
+    curl -fL -w "\n" --upload-file - \
+    -H "Name: $BIG" \
+    -H "Max-Downloads: 4" \
+    -H "Expires-Days: 1" \
+    -H "Secret: $FLASK_ACCESS_TOKEN" \
+    "$FLASK_PUBLIC_URL"/upload
+}
+
 function t05-check-db-manually() {
   sqlite3 ../data/flees.db "select * FROM files"
 }
@@ -249,6 +259,9 @@ function t13-mfl-list() {
 function t14-mfl-upload() {
   ./mfl w mfl
   ./mfl w -d 1 -m 1 mfl
+  cat mfl | ./mfl w mfl
+  ./mfl w . "folder with spaces"
+  ./mfl w "$SMALL"
 }
 
 
@@ -279,8 +292,10 @@ _getchoice() {
 }
 
 
-next_task=0
+if [[ -z "$1" ]]; then
-while true; do
+
+  next_task=0
+  while true; do
     choice=$( _getchoice $next_task )
     if [[ -z "$choice" ]]; then break; fi
     _title "$choice"
@@ -291,7 +306,23 @@ while true; do
     #~ next_task=$(( next_task + 1 ))
     next_task=$( _getnext "$choice" )
 
-done
+  done
 
+else
+  # user passed commands from cmd line
+  if [[ "$1" = "all" ]]; then
+    args=$( _getlist )
+  else
+    args="$@"
+  fi
 
+  for choice in $args; do
+    _title "$choice"
+    set -x
+    "$choice"
+    set +x
+    _title ""
+    if [[ "$choice" = "t00-rebuild-docker" ]]; then sleep 3; fi
+  done
+fi