hidden files, allow ips

code/app.py

@@ -1,44 +1,51 @@
 # -*- coding: utf-8 -*-
 
+import logging
 import os
 import sys
 import time
+
 from flask import (
     Flask,
-    render_template,
     jsonify,
-    request,
-    url_for,
     redirect,
+    render_template,
+    request,
     send_from_directory,
     session,
+    url_for,
 )
-from werkzeug.utils import secure_filename
 from revprox import ReverseProxied
-from utils.misc import random_token, hash_password, verify_password, file_date_human
 from utils.files import (
+    db_add_download,
+    db_delete_file,
+    db_get_file,
+    db_maintenance,
     db_store_file,
     file_details,
-    file_list,
-    file_list_simple,
     file_full_path,
     file_full_url,
-    db_add_download,
+    file_list,
-    db_get_file,
+    file_list_simple,
-    db_delete_file,
-    db_maintenance,
-    validate_upload_token,
     invalidate_upload_token,
     new_upload_token,
+    validate_upload_token,
 )
-import logging
+from utils.misc import (
+    file_date_human,
+    hash_password,
+    is_ip_allowed,
+    random_token,
+    verify_password,
+)
+from werkzeug.utils import secure_filename
 
 logging.basicConfig(
     level=logging.INFO,
     format=f"[%(asctime)s] [%(levelname)s] %(message)s",
 )
 
-__VERSION__ = "20230828.0"
+__VERSION__ = "20240114.0"
 app = Flask(__name__)
 app.config.from_object(__name__)
 app.config.from_prefixed_env()
@@ -81,6 +88,8 @@ def upload():
         -H "Max-Downloads: 4000" \
         -H "Expires-Days: 14" \
         -H "Password: mypass" \
+        -H "Hidden: true" \
+        -H "Allowed-IP: 10.0.0.1,10.0.0.2,10.1.*" \
         -H "Secret: dff789f0bbe8183d32542" \
         "$FLASK_PUBLIC_URL"/upload
 
@@ -115,15 +124,21 @@ def upload():
     max_dl = request.headers.get("Max-Downloads", app.config["DEFAULT_MAX_DL"])
     expires = int(time.time()) + int(app.config["DEFAULT_EXPIRE"])
     if "Expires-days" in request.headers:
-        expires = int(time.time()) + 24 * 3600 * int(
+        expires = int(time.time()) + 24 * 3600 * int(request.headers.get("Expires-days"))
-            request.headers.get("Expires-days")
-        )
     if "Expires-hours" in request.headers:
         expires = int(time.time()) + 3600 * int(request.headers.get("Expires-hours"))
     password = None
     if "Password" in request.headers:
         if request.headers["Password"] != "":
             password = hash_password(request.headers["Password"])
+    hidden = None
+    if "Hidden" in request.headers:
+        hidden = request.headers["Hidden"].lower() == "true"
+    allowed_ip = None
+    if "Allowed-IP" in request.headers:
+        allowed_ip = request.headers["Allowed-IP"].lower()
+        if not set(allowed_ip) <= set("1234567890.*, "):
+            return "IP list contains unknown characters"
 
     while True:
         token = random_token()
@@ -150,11 +165,9 @@ def upload():
                     break
                 f.write(chunk)
 
-    db_store_file(token, safe_filename, expires, max_dl, password)
+    db_store_file(token, safe_filename, expires, max_dl, password, ips=allowed_ip, hidden=hidden)
     download_url = file_full_url(token, safe_filename)
-    app.logger.info(
+    app.logger.info(f"Upload: {download_url} MaxDL:{max_dl} Exp:{file_date_human(expires)}")
-        f"Upload: {download_url} MaxDL:{max_dl} Exp:{file_date_human(expires)}"
-    )
     if upload_token:
         invalidate_upload_token(upload_token)
     return "File uploaded\n%s\n" % (download_url,), 200
@@ -176,9 +189,7 @@ def upload_token():
         return "Error", 401
     expires = int(time.time()) + int(app.config["DEFAULT_EXPIRE"])
     if "Expires-days" in request.headers:
-        expires = int(time.time()) + 24 * 3600 * int(
+        expires = int(time.time()) + 24 * 3600 * int(request.headers.get("Expires-days"))
-            request.headers.get("Expires-days")
-        )
     token = new_upload_token(expires)
     return token, 200
 
@@ -197,6 +208,11 @@ def details(token, name):
     if secret != app.config["ACCESS_TOKEN"]:
         return "Error", 401
     details = file_details(token, name)
+    if details["allowed_ip"] is not None:
+        if not is_ip_allowed(
+            details["allowed_ip"], request.environ.get("HTTP_X_FORWARDED_FOR", request.remote_addr), app
+        ):
+            return "Not Allowed", 403
     return jsonify(details), 200
 
 
@@ -208,6 +224,13 @@ def delete_file(name, token):
     secret = request.headers.get("Secret", "")
     if secret != app.config["ACCESS_TOKEN"]:
         return "Error", 401
+    details = file_details(token, name)
+    if details["allowed_ip"] is not None:
+        if not is_ip_allowed(
+            details["allowed_ip"], request.environ.get("HTTP_X_FORWARDED_FOR", request.remote_addr), app
+        ):
+            return "Not Allowed", 403
+
     try:
         os.remove(os.path.join(app.config["DATAFOLDER"], token, name))
     except Exception:
@@ -307,13 +330,17 @@ def download_file(token, name):
         return "Error", 404
     db_stat = db_get_file(token, name)
     if db_stat:
-        added, expires, downloads, max_dl, password_hash = db_stat
+        added, expires, downloads, max_dl, password_hash, hidden, allowed_ip = db_stat
     else:
         return "Error", 404
+    if allowed_ip is not None:
+        if not is_ip_allowed(allowed_ip, request.environ.get("HTTP_X_FORWARDED_FOR", request.remote_addr), app):
+            return "Not Allowed", 403
+    #  check ip list!
     if downloads >= max_dl and max_dl > -1:
-        return "Expired", 401
+        return "Expired", 403
     if expires < time.time():
-        return "Expired", 401
+        return "Expired", 403
     if password_hash:
         if verify_password(session.get(token, ""), password_hash):
             pass
@@ -324,9 +351,7 @@ def download_file(token, name):
             return redirect(url_for("login"))
 
     db_add_download(token, name)
-    return send_from_directory(
+    return send_from_directory(directory=os.path.join(app.config["DATAFOLDER"], token), path=name)
-        directory=os.path.join(app.config["DATAFOLDER"], token), path=name
-    )
 
 
 def print_debug(s):

code/init_db.sh

@@ -9,7 +9,9 @@ CREATE TABLE IF NOT EXISTS files (
     expires       integer NOT NULL,
     downloads     integer NOT NULL,
     max_downloads integer NOT NULL,
-    passhash      text
+    passhash      text,
+    allowed_ip    text,
+    hidden        boolean
 );
 EOF
 

code/templates/mfl

@@ -17,6 +17,8 @@ _help() {
     -d  max days to keep      (default 30)
     -m  max downloads to keep (default 9999,  -1 to disable)
     -p  password
+    --hidden  hidden file (from listings)
+    --allow   comma separated list of ip addresses, accepts *
 
   Filename:
     When writing to share:
@@ -88,7 +90,7 @@ _write() {
 
   [[ -d "$FILE" ]] && {
     [[ "${NAME,,}" = *".tar" ]] || NAME="${NAME}".tar
-    _write_folder "$NAME" "$FILE"
+    tar c "$2" | _write_stdin "$NAME"
     return $?
   }
 
@@ -96,28 +98,9 @@ _write() {
       _msg "No such file"
       exit 1
   }
-  _write_file "$NAME" "$FILE"
+  cat "$FILE" | _write_stdin "$NAME"
 }
 
-_write_folder() { # name, file
-  tar c "$2" | \
-  curl -fL -w "\n" -g --upload-file - \
-    -H "Name: $1" \
-    -H "Max-Downloads: $MAXDL" \
-    -H "Expires-Days: $MAXDAYS" \
-    -H "Secret: $MFL_TOKEN" \
-    -H "Password: $PASSWORD" \
-    "$MFL_ROOTURL"/upload | cat
-}
-_write_file() { # name, file
-  curl -fL -w "\n" -g --upload-file "$2" \
-    -H "Name: $1" \
-    -H "Max-Downloads: $MAXDL" \
-    -H "Expires-Days: $MAXDAYS" \
-    -H "Secret: $MFL_TOKEN" \
-    -H "Password: $PASSWORD" \
-    "$MFL_ROOTURL"/upload | cat
-}
 _write_stdin() { # name
   cat - | \
   curl -fL -w "\n" -g --upload-file - \
@@ -125,7 +108,9 @@ _write_stdin() { # name
     -H "Max-Downloads: $MAXDL" \
     -H "Expires-Days: $MAXDAYS" \
     -H "Secret: $MFL_TOKEN" \
+    -H "Hidden: $HIDDEN" \
     -H "Password: $PASSWORD" \
+    -H "Allowed-IP: $ALLOWED" \
     "$MFL_ROOTURL"/upload | cat
 }
 
@@ -218,6 +203,8 @@ done
 
 MAXDL=9999
 MAXDAYS=30
+HIDDEN=false
+ALLOWED=""
 CMD=list
 # if stdin comes from stream, default to write
 [ -t 0 ] || CMD=help
@@ -226,6 +213,8 @@ for (( i=2; i<=$#; i++ )); do
   [[ "${!i}" = "-m" ]] && { MAXDL=${!j}; i=$j; continue; }
   [[ "${!i}" = "-d" ]] && { MAXDAYS=${!j}; i=$j; continue; }
   [[ "${!i}" = "-p" ]] && { PASSWORD=${!j}; i=$j; continue; }
+  [[ "${!i}" = "--allow" ]] && { ALLOWED="${!j}"; i=$j; continue; }
+  [[ "${!i}" = "--hidden" ]] && { HIDDEN=true; continue; }
   if [[ -z "$FILE" ]]; then
     FILE="${!i}"
     continue

code/utils/files.py

@@ -1,10 +1,12 @@
 import os
-from datetime import datetime
-from flask import current_app as app
-import time
-import sqlite3
 import shutil
-from .misc import file_size_human, file_date_human, file_time_human, random_token
+import sqlite3
+import time
+from datetime import datetime
+
+from flask import current_app as app
+
+from .misc import file_date_human, file_size_human, file_time_human, random_token
 
 
 def get_db():
@@ -13,14 +15,14 @@ def get_db():
     return db, c
 
 
-def db_store_file(token, name, expires, max_dl, password):
+def db_store_file(token, name, expires, max_dl, password, ips=None, hidden=None):
     db, c = get_db()
     c.execute(
         """
-        insert into files(token,name,added,expires,downloads,max_downloads,passhash)
+        insert into files(token,name,added,expires,downloads,max_downloads,passhash,allowed_ip,hidden)
-                values (?,?,?,?,?,?,?)
+                values (?,?,?,?,?,?,?,?,?)
                 """,
-        (token, name, int(time.time()), expires, 0, max_dl, password),
+        (token, name, int(time.time()), expires, 0, max_dl, password, ips, hidden),
     )
     if c.rowcount > 0:
         db.commit()
@@ -31,7 +33,7 @@ def db_get_file(token, name):
     db, c = get_db()
     return db.execute(
         """
-        SELECT added,expires,downloads,max_downloads,passhash
+        SELECT added,expires,downloads,max_downloads,passhash,hidden,allowed_ip
         FROM files WHERE token = ? AND name = ?
                 """,
         (token, name),
@@ -42,9 +44,11 @@ def db_get_files():
     db, c = get_db()
     return db.execute(
         """
-        SELECT token,name,added,expires,downloads,max_downloads,passhash
+        SELECT token,name,added,expires,downloads,max_downloads,passhash,allowed_ip
         FROM files
-        WHERE expires > ? and (downloads < max_downloads or max_downloads = -1)
+        WHERE expires > ?
+              AND (downloads < max_downloads or max_downloads = -1)
+              AND (hidden IS NULL OR hidden = 0)
         ORDER BY added
         """,
         (time.time(),),
@@ -168,8 +172,7 @@ def file_age(path):
     diff = now - then
     return (
         diff,
-        "%03d d %s"
+        "%03d d %s" % (diff.days, datetime.utcfromtimestamp(diff.seconds).strftime("%H:%M:%S")),
-        % (diff.days, datetime.utcfromtimestamp(diff.seconds).strftime("%H:%M:%S")),
     )
 
 
@@ -179,7 +182,7 @@ def file_details(token, name):
         s = os.stat(full_path)
         db_stat = db_get_file(token, name)
         if db_stat:
-            added, expires, downloads, max_dl, password = db_stat
+            added, expires, downloads, max_dl, password, hidden, allowed_ip = db_stat
         else:
             return {}
         return {
@@ -192,6 +195,8 @@ def file_details(token, name):
             "downloaded": downloads,
             "max-dl": max_dl,
             "protected": password is not None,
+            "hidden": hidden,
+            "allowed_ip": allowed_ip,
         }
     except FileNotFoundError:
         return {}
@@ -214,7 +219,9 @@ def file_list():
         added = file_date_human(file[2])
         expiry = file_date_human(file[3])
         pw = " (PW)" if file[6] else ""
-        details.append(f"{added}/{expiry} {file[4]:4d}/{file[5]:4d} {url}{pw}")
+        ips = f" [{file[7]}]" if file[7] else ""
+
+        details.append(f"{added}/{expiry} {file[4]:4d}/{file[5]:4d} {url}{pw}{ips}")
 
     return details
 

code/utils/misc.py

@@ -1,6 +1,8 @@
-from datetime import datetime
+import fnmatch
 import secrets
 import string
+from datetime import datetime
+
 import passlib.hash
 
 VALID_TOKEN_CHARS = string.digits + string.ascii_letters
@@ -38,3 +40,11 @@ def hash_password(password):
 
 def verify_password(password, hash):
     return passlib.hash.argon2.verify(password, hash)
+
+
+def is_ip_allowed(allowed_ip, ip, app):
+    for rule in allowed_ip.split(","):
+        rule = rule.strip()
+        if fnmatch.fnmatch(ip, rule):
+            return True
+    return False

test/run-tests.sh

@@ -87,6 +87,12 @@ _title() {
 
 set -e
 
+if [[ ! -d ../test ]]; then
+   echo run in the test folder
+   exit 1
+fi
+
+
 BIG="big file(1).ext"
 BIGS="big_file1.ext"
 SMALL="small file"
@@ -316,7 +322,54 @@ function t17-new-upload_token() {
 
 }
 
+function t18-upload_hidden() {
+  url=$( cat "$IMAGE" | \
+    curl -fL -w "\n" -F file="@-" -X POST \
+    -H "Name: $IMAGE" \
+    -H "Secret: $FLASK_ACCESS_TOKEN" \
+    -H "Hidden: true" \
+    -H "Max-Downloads: 1" \
+    "$FLASK_PUBLIC_URL"/upload | grep ^http )
 
+  curl "$url" > /dev/null
+
+  ./mfl w -m 1 --hidden "$IMAGE"
+
+  curl -fL -w "\n" \
+    -H "Secret: $FLASK_ACCESS_TOKEN" \
+    "$FLASK_PUBLIC_URL"/ls
+}
+
+
+function t19-upload_ip_allow() {
+  url=$( cat "$IMAGE" | \
+    curl -fL -w "\n" -F file="@-" -X POST \
+    -H "Name: $IMAGE" \
+    -H "Secret: $FLASK_ACCESS_TOKEN" \
+    -H "Allowed-IP: *" \
+    "$FLASK_PUBLIC_URL"/upload | grep ^http )
+  curl -s -D /dev/stderr "$url" | head -c 1 | head -c 0
+
+  url=$( cat "$IMAGE" | \
+    curl -fL -w "\n" -F file="@-" -X POST \
+    -H "Name: $IMAGE" \
+    -H "Secret: $FLASK_ACCESS_TOKEN" \
+    -H "Allowed-IP: 10.0.0.10, 172.20.*" \
+    "$FLASK_PUBLIC_URL"/upload | grep ^http )
+  curl -s -D /dev/stderr "$url" | head -c 1 | head -c 0
+
+  url=$( cat "$IMAGE" | \
+    curl -fL -w "\n" -F file="@-" -X POST \
+    -H "Name: $IMAGE" \
+    -H "Secret: $FLASK_ACCESS_TOKEN" \
+    -H "Allowed-IP: 10.0.0.10, 12.12.12.12" \
+    "$FLASK_PUBLIC_URL"/upload | grep ^http )
+  curl -s -D /dev/stderr "$url" | head -c 1 | head -c 0
+
+  ./mfl w -m 1 --allow '*' "$IMAGE"
+  ./mfl w -m 1 --allow '10.0.0.1, 10.0.0.*' "$IMAGE"
+  ./mfl list
+}
 
 _getlist() {
   declare -F | awk '{ print $3 }' | grep -v ^_