upload with PUT allows streaming

code/app.py

@@ -27,7 +27,7 @@ from utils.files import (
     db_maintenance,
 )
 
-__VERSION__ = "20230819.0"
+__VERSION__ = "20230820.0"
 app = Flask(__name__)
 app.config.from_object(__name__)
 app.config.from_prefixed_env()
@@ -42,7 +42,7 @@ def index():
     return "", 200
 
 
-@app.route("/upload", methods=["POST"])
+@app.route("/upload", methods=["PUT","POST"])
 def upload():
     """
     Upload a file, example CURL:
@@ -58,45 +58,64 @@ def upload():
     - Additionally, "Expires-Hours" can be used.
     - Max-Dowloads: -1  means no upper limit
 
+    IF using GET, you can upload larger files with pipes
+
+    cat largefile | \
+        curl -fL -w "\n" --upload-file - \
+        -H "Name: my.file.ext" \
+        -H "Max-Downloads: 4000" \
+        -H "Expires-Days: 14" \
+        -H "Secret: dff789f0bbe8183d32542" \
+        "$FLASK_PUBLIC_URL"/upload
 
     Returns the file download URL
     """
 
+    name = request.headers.get("Name", None)
+    if name is None:
+        return "Name required", 500
+    safe_filename = secure_filename(name)
+    secret = request.headers.get("Secret", "")
+    if secret != app.config["ACCESS_TOKEN"]:
+        return "Error", 401
+    max_dl = request.headers.get("Max-Downloads", app.config["DEFAULT_MAX_DL"])
+    expires = int(time.time()) + int(app.config["DEFAULT_EXPIRE"])
+    if "Expires-days" in request.headers:
+        expires = int(time.time()) + 24 * 3600 * int(
+            request.headers.get("Expires-days")
+        )
+    if "Expires-hours" in request.headers:
+        expires = int(time.time()) + 3600 * int(
+            request.headers.get("Expires-hours")
+        )
+
+    while True:
+        token = random_token()
+        folder = os.path.join(app.config["DATAFOLDER"], token)
+        if not os.path.exists(folder):
+            break
+    filename = file_full_path(token, safe_filename)
+    os.mkdir(folder)
+
     if request.method == "POST":
         file = request.files.get("file")
-        name = request.headers.get("Name", None)
-        if name is None:
-            return "Name required", 500
-        secret = request.headers.get("Secret", "")
-        if secret != app.config["ACCESS_TOKEN"]:
-            return "Error", 401
-        max_dl = request.headers.get("Max-Downloads", app.config["DEFAULT_MAX_DL"])
-        expires = int(time.time()) + int(app.config["DEFAULT_EXPIRE"])
-        if "Expires-days" in request.headers:
-            expires = int(time.time()) + 24 * 3600 * int(
-                request.headers.get("Expires-days")
-            )
-        if "Expires-hours" in request.headers:
-            expires = int(time.time()) + 3600 * int(
-                request.headers.get("Expires-hours")
-            )
-
         if file:
-            safe_filename = secure_filename(name)
-            while True:
-                token = random_token()
-                folder = os.path.join(app.config["DATAFOLDER"], token)
-                if not os.path.exists(folder):
-                    break
-            os.mkdir(folder)
-            filename = file_full_path(token, safe_filename)
             file.save(filename)
-            db_store_file(token, safe_filename, expires, max_dl)
-            download_url = file_full_url(token, safe_filename)
-            return "File uploaded\n%s\n" % (download_url,), 200
         else:
             return "Use the 'file' variable to upload\n", 400
 
+    if request.method == "PUT":
+        chunk_size = 1024 * 1024 * 64 # 64Mb
+        with open(filename, 'wb') as f:
+            while True:
+                chunk = request.stream.read(chunk_size)
+                if not chunk:
+                  break
+                f.write(chunk)
+
+    db_store_file(token, safe_filename, expires, max_dl)
+    download_url = file_full_url(token, safe_filename)
+    return "File uploaded\n%s\n" % (download_url,), 200
 
 @app.route("/details/<token>/<name>", methods=["GET"])
 def details(token, name):

code/entrypoint.sh

@@ -8,6 +8,7 @@ export FLASK_CONF="/data/config.json"
 export SERVER=gunicorn
 export PID="flees.pid"
 export WORKERS
+export TIMEOUT
 
 if [[ $( stat -c %u /data ) -ne $( id -u ) ]]; then
   echo User id and /data folder owner do not match
@@ -19,4 +20,4 @@ set -eu
 . /opt/venv/bin/activate
 sh ./init_db.sh "$FLASK_DB"
 
-exec "$SERVER" -w $WORKERS 'app:app' --pid="$PID" -b 0.0.0.0:5000
+exec "$SERVER" -w $WORKERS --timeout $TIMEOUT 'app:app' --pid="$PID" -b 0.0.0.0:5000

code/templates/mfl

@@ -104,8 +104,7 @@ _write() {
 
 _write_folder() { # name, file
   tar c "$2" | \
-  curl -fL -w "\n" -F file="@-" -X POST \
+  curl -fL -w "\n" -g --upload-file - \
-    --progress-bar \
     -H "Name: $1" \
     -H "Max-Downloads: $MAXDL" \
     -H "Expires-Days: $MAXDAYS" \
@@ -113,8 +112,7 @@ _write_folder() { # name, file
     "$MFL_ROOTURL"/upload | cat
 }
 _write_file() { # name, file
-  curl -fL -w "\n" -F file="@$2" -X POST \
+  curl -fL -w "\n" -g --upload-file "$2" \
-    --progress-bar \
     -H "Name: $1" \
     -H "Max-Downloads: $MAXDL" \
     -H "Expires-Days: $MAXDAYS" \
@@ -123,8 +121,7 @@ _write_file() { # name, file
 }
 _write_stdin() { # name
   cat - | \
-  curl -fL -w "\n" -F file="@-" -X POST \
+  curl -fL -w "\n" -g --upload-file - \
-    --progress-bar \
     -H "Name: $1" \
     -H "Max-Downloads: $MAXDL" \
     -H "Expires-Days: $MAXDAYS" \
@@ -211,7 +208,7 @@ for (( i=2; i<=$#; i++ )); do
 done
 if [[ -z "$NAME" ]]; then
   if [[ -n "$FILE" ]]; then
-      NAME="$( basename ${FILE} )"
+      NAME="$( basename "${FILE}" )"
   fi
 fi
 

docker-compose.yaml

@@ -15,7 +15,8 @@ services:
       - ./data/:/data/
     restart: unless-stopped
     environment:
-      WORKERS: ${WORKERS}
+      WORKERS:
+      TIMEOUT:
       FLASK_APP_SECRET_KEY:
       FLASK_ACCESS_TOKEN:
       FLASK_PUBLIC_URL:

env.example

@@ -3,6 +3,7 @@ UID=1000
 GID=1000
 TZ=Europe/Helsinki
 WORKERS=4
+TIMEOUT=1800
 FLASK_APP_SECRET_KEY=8a36bfea77d842386a2a0c7c3e044228363dfddadc01fade4b1b78859ffc615b
 FLASK_ACCESS_TOKEN=dff789f0bbe8183d3254258b33a147d580c1131f39a698c56d3f640ac8415714
 FLASK_PUBLIC_URL=http://localhost:8136

test/run-tests.sh

@@ -103,7 +103,7 @@ test -f "$IMAGE" || convert -size 640x480 xc:gray $IMAGE
 function t00-rebuild-docker() {
   CWD=$PWD
   cd ..
-  docker-compose up --build -d --force-recreate
+  docker-compose up --build -d --force-recreate -t 0
   cd "$CWD"
 }
 
@@ -133,7 +133,7 @@ function t03-upload-small-file() {
 
 }
 
-function t04-upload-large-file() {
+function t04-upload-large-file-POST() {
   pv "$BIG" | \
     curl -fL -w "\n" -F file="@-" -X POST \
     -H "Name: $BIG" \
@@ -143,6 +143,16 @@ function t04-upload-large-file() {
     "$FLASK_PUBLIC_URL"/upload
 }
 
+function t04-upload-large-file-GET() {
+  pv "$BIG" | \
+    curl -fL -w "\n" --upload-file - \
+    -H "Name: $BIG" \
+    -H "Max-Downloads: 4" \
+    -H "Expires-Days: 1" \
+    -H "Secret: $FLASK_ACCESS_TOKEN" \
+    "$FLASK_PUBLIC_URL"/upload
+}
+
 function t05-check-db-manually() {
   sqlite3 ../data/flees.db "select * FROM files"
 }
@@ -233,22 +243,25 @@ function t10-maintenance-post() {
 }
 
 function t11-get-mfl() {
-    curl -fL  \
+  curl -fL  \
     -H "Secret: $FLASK_ACCESS_TOKEN" \
     "$FLASK_PUBLIC_URL"/script/mfl > mfl
-    chmod +x mfl
+  chmod +x mfl
 
 }
 function t12-mfl-update() {
-    ./mfl update
+  ./mfl update
 }
 function t13-mfl-list() {
-    ./mfl list
+  ./mfl list
-    ./mfl
+  ./mfl
 }
 function t14-mfl-upload() {
-    ./mfl w mfl
+  ./mfl w mfl
-    ./mfl w -d 1 -m 1 mfl
+  ./mfl w -d 1 -m 1 mfl
+  cat mfl | ./mfl w mfl
+  ./mfl w . "folder with spaces"
+  ./mfl w "$SMALL"
 }
 
 
@@ -279,19 +292,37 @@ _getchoice() {
 }
 
 
-next_task=0
+if [[ -z "$1" ]]; then
-while true; do
-  choice=$( _getchoice $next_task )
-  if [[ -z "$choice" ]]; then break; fi
-  _title "$choice"
-  set -x
-  "$choice"
-  set +x
-  _title ""
-  #~ next_task=$(( next_task + 1 ))
-  next_task=$( _getnext "$choice" )
 
-done
+  next_task=0
+  while true; do
+    choice=$( _getchoice $next_task )
+    if [[ -z "$choice" ]]; then break; fi
+    _title "$choice"
+    set -x
+    "$choice"
+    set +x
+    _title ""
+    #~ next_task=$(( next_task + 1 ))
+    next_task=$( _getnext "$choice" )
 
+  done
 
+else
+  # user passed commands from cmd line
+  if [[ "$1" = "all" ]]; then
+    args=$( _getlist )
+  else
+    args="$@"
+  fi
+
+  for choice in $args; do
+    _title "$choice"
+    set -x
+    "$choice"
+    set +x
+    _title ""
+    if [[ "$choice" = "t00-rebuild-docker" ]]; then sleep 3; fi
+  done
+fi