commit 2157306a4e382c555c708b1af56f715c413bd351 Author: Ville Rantanen Date: Thu Aug 18 15:42:27 2022 +0300 initial diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bae06e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +data/** diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..14893d8 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,24 @@ +version: "3.9" + +services: + pwss: + build: + context: docker-pwss + args: + - UUID + - TZ + image: pwss + volumes: + - ./data/:/data/ + environment: + - UUID + - SESSION_EXPIRY + - SECRET_KEY + - DATABASE + - WORKERS + - CONFIG_FOLDER + - STATIC_FOLDER + - LIMITER_SHARE + ports: + - "${EXPOSE}:5000" + restart: "unless-stopped" diff --git a/docker-pwss/Dockerfile b/docker-pwss/Dockerfile new file mode 100644 index 0000000..c441107 --- /dev/null +++ b/docker-pwss/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.10-slim-bullseye + +ARG TZ=UTC +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN apt-get update && apt-get upgrade -y && apt-get install -y \ + memcached \ + && rm -rf /var/lib/apt/lists/ + +COPY code/requirements.txt /tmp/requirements.txt + +ARG UUID=1000 +RUN useradd -u $UUID -ms /bin/bash user && \ + mkdir /venv && chown $UUID /venv +USER user +RUN python3 -m venv /venv && \ + . /venv/bin/activate && \ + pip install --no-cache-dir -U pip wheel && \ + pip install --no-cache-dir -r /tmp/requirements.txt && \ + rm -rf /home/user/.cache && pip freeze > /venv/freeze.txt + +EXPOSE 5000 +EXPOSE 5001 +COPY code /code +WORKDIR /code +CMD ["bash", "/code/start_daemon"] diff --git a/docker-pwss/code/maintain b/docker-pwss/code/maintain new file mode 100755 index 0000000..200fe57 --- /dev/null +++ b/docker-pwss/code/maintain @@ -0,0 +1,5 @@ +#!/bin/bash + +. /venv/bin/activate +set -x +exec python3 maintain.py diff --git a/docker-pwss/code/maintain.py b/docker-pwss/code/maintain.py new file mode 100644 index 0000000..c367b59 --- /dev/null +++ b/docker-pwss/code/maintain.py @@ -0,0 +1,22 @@ + +from share import session_clean, session_create_database +import sys +import time + + +def run_periodically(): + + while True: + try: + print("Cleaning sessions", file=sys.stderr) + session_clean() + except Exception as e: + print(e, file=sys.stderr) + + # Run daily + time.sleep(86400) + + +if __name__ == "__main__": + session_create_database() + run_periodically() diff --git a/docker-pwss/code/manager b/docker-pwss/code/manager new file mode 100755 index 0000000..8de84f4 --- /dev/null +++ b/docker-pwss/code/manager @@ -0,0 +1,5 @@ +#!/bin/bash + +cd $( dirname $( readlink -f "$0" ) ) +. /venv/bin/activate +python3 /code/share.py "$@" diff --git a/docker-pwss/code/requirements.txt b/docker-pwss/code/requirements.txt new file mode 100644 index 0000000..ef0a235 --- /dev/null +++ b/docker-pwss/code/requirements.txt @@ -0,0 +1,6 @@ +gunicorn +flask +flask-limiter[memcached] +bcrypt +pymemcache +supervisor diff --git a/docker-pwss/code/revprox.py b/docker-pwss/code/revprox.py new file mode 100644 index 0000000..3914560 --- /dev/null +++ b/docker-pwss/code/revprox.py @@ -0,0 +1,33 @@ +class ReverseProxied(object): + """Wrap the application in this middleware and configure the + front-end server to add these headers, to let you quietly bind + this to a URL other than / and to an HTTP scheme that is + different than what is used locally. + + In nginx: + location /myprefix { + proxy_pass http://192.168.0.1:5001; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Script-Name /myprefix; + } + + :param app: the WSGI application + """ + + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + script_name = environ.get("HTTP_X_SCRIPT_NAME", "") + if script_name: + environ["SCRIPT_NAME"] = script_name + path_info = environ["PATH_INFO"] + if path_info.startswith(script_name): + environ["PATH_INFO"] = path_info[len(script_name) :] + + scheme = environ.get("HTTP_X_SCHEME", "") + if scheme: + environ["wsgi.url_scheme"] = scheme + return self.app(environ, start_response) diff --git a/docker-pwss/code/serve b/docker-pwss/code/serve new file mode 100755 index 0000000..dbba8c8 --- /dev/null +++ b/docker-pwss/code/serve @@ -0,0 +1,9 @@ +#!/bin/bash + +. /venv/bin/activate +set -x +exec gunicorn \ + -b 0.0.0.0:5000 \ + -w "$WORKERS" \ + serve:app + diff --git a/docker-pwss/code/serve.py b/docker-pwss/code/serve.py new file mode 100644 index 0000000..a749fa8 --- /dev/null +++ b/docker-pwss/code/serve.py @@ -0,0 +1,128 @@ +from flask import ( + Flask, + request, + session, + g, + redirect, + url_for, + render_template, + send_file, +) + +from flask_limiter import Limiter +from revprox import ReverseProxied +from utils import ( + authenticate, + check_auth, + get_ip, + get_valid_sessions, + read_config, +) +import os +import sys + +DEBUG = False +SECRET_KEY = os.getenv("SECRET_KEY", "2f6aa45dfcfc37a50537f0b05af6452c") +DATABASE = os.getenv("DATABASE", "serve.db") +SESSION_EXPIRY = int(os.getenv("SESSION_EXPIRY", 1800)) +FOLDERS = os.getenv("STATIC_FOLDER") + +app = Flask(__name__) +app.config.from_object(__name__) +app.wsgi_app = ReverseProxied(app.wsgi_app) + +limiter = Limiter( + key_func=get_ip, + default_limits=["5 per 5 seconds"], + storage_uri="memcached://localhost:11211", +) +limiter.init_app(app) + + +@app.route("/s/", methods=["POST", "GET"]) +@limiter.limit(os.getenv("LIMITER_SHARE")) +def serve(path=None): + realpath = os.path.join(FOLDERS, path) + is_auth = check_auth(path) + if not is_auth: + session["return_to"] = path + return redirect( + url_for("login", folder=path.split(os.sep)[0]), code=302 + ) + + if os.path.isdir(realpath): + if not path.endswith("/"): + return redirect(url_for("serve", path=f"{path}/"), code=302) + realpath = os.path.join(realpath, "index.html") + + if not os.path.exists(realpath): + return "", 404 + + return send_file( + realpath, + ) + + +@app.route("/l", methods=["POST", "GET"]) +@app.route("/l/", methods=["POST", "GET"]) +def login(folder=None): + + if request.method == "POST": + folder = "".join( + letter for letter in request.form["folder"] if letter.isalnum() + ) + config = read_config(folder) + success = authenticate(config, request.form["password"]) + print( + f"{'Successful' if success else 'Failed'} login {folder}: {get_ip()}", + file=sys.stderr, + ) + + ret = session.get("return_to", None) + + if "return_to" in session: + session.pop("return_to") + if ret: + return redirect(url_for("serve", path=ret)) + else: + # GET + config = read_config(folder) + + sessions = get_valid_sessions() + if not folder: + folder = "" + return render_template("login.html", folder=folder, sessions=sessions) + + +@app.route("/logout", methods=["GET"]) +def logout(): + to_delete = [key for key in session if key.startswith("auth/")] + for key in to_delete: + del session[key] + return redirect(url_for("index")) + + +@app.route("/", methods=["GET"]) +def index(): + return render_template("index.html") + + +@app.teardown_appcontext +def close_connection(exception): + db = getattr(g, "_database", None) + if db is not None: + db.close() + + +@app.errorhandler(429) +def ratelimit_handler(e): + print( + f"Ratelimit exceeded: {e.description} key: {e.limit.key_func()}", + file=sys.stderr, + ) + return render_template("ratelimit.html", description=e.description), 429 + + +if __name__ == "__main__": + + app.run() diff --git a/docker-pwss/code/share.py b/docker-pwss/code/share.py new file mode 100644 index 0000000..dcb6813 --- /dev/null +++ b/docker-pwss/code/share.py @@ -0,0 +1,222 @@ +import os +import sys +import json +import time +import bcrypt +import argparse +import sqlite3 +from werkzeug.utils import secure_filename +from datetime import datetime, timedelta +from utils import read_config + +ENTRY = {"expires": "never", "password": None} +CONFIGS = os.getenv("CONFIG_FOLDER") +FOLDERS = os.getenv("STATIC_FOLDER") +DATABASE = os.getenv("DATABASE", None) +SCHEMA = """CREATE TABLE IF NOT EXISTS sessions( + ID INTEGER PRIMARY KEY AUTOINCREMENT, + folder TEXT NOT NULL, + ip TEXT NOT NULL, + token TEXT NOT NULL, + expire INTEGER NOT NULL +);""" + + +def get_opts(): + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command") + sub_add = subparsers.add_parser("add") + sub_remove = subparsers.add_parser("remove") + sub_edit = subparsers.add_parser("edit") + sub_list = subparsers.add_parser("list") + sub_session_list = subparsers.add_parser("sessions-list") + sub_session_clean = subparsers.add_parser( + "sessions-clean", help="Clean out session DB" + ) + sub_session_remove = subparsers.add_parser( + "sessions-remove", help="Terminates all sessions" + ) + + for p in (sub_add, sub_remove, sub_edit): + p.add_argument("folder", action="store", help="Folder name to share") + + for p in (sub_add, sub_edit): + + p.add_argument( + "--expires", + action="store", + help="Share expires in days, or 'never'", + default=None, + ) + p.add_argument( + "--password", action="store", help="Set password", default=None + ) + + args = parser.parse_args() + return args + + +def manager(): + opts = get_opts() + if opts.command == None or opts.command == "list": + shares_list() + if opts.command == "add": + share_add(opts) + if opts.command == "remove": + share_remove(opts) + if opts.command == "edit": + share_edit(opts) + if opts.command == "sessions-list": + session_list() + if opts.command == "sessions-remove": + session_remove() + if opts.command == "sessions-clean": + session_clean() + + +def load_config(name): + config = read_config(name) + return config + + +def save_config(path, config): + with open(path, "wt") as fp: + save_config = ENTRY.copy() + for key in save_config: + save_config[key] = config[key] + return json.dump(save_config, fp, indent=2, sort_keys=True) + + +def share_oneliner(entry): + return f"{(entry['name']+'/').ljust(15)} expires: {entry['expires']}" + + +def shares_list(): + + print("Shared folders:") + for c in sorted(os.listdir(CONFIGS)): + if c.endswith(".json"): + entry = load_config(c[0:-5]) + print(share_oneliner(entry)) + + +def share_add(opts): + + entry = ENTRY.copy() + if opts.folder != secure_filename(opts.folder): + raise ValueError(f"Folder '{opts.folder}' is not a safe filename") + + if opts.expires: + entry["expires"] = ( + (datetime.now() + timedelta(days=float(opts.expires))) + .replace(microsecond=0) + .isoformat() + ) + if not opts.password: + raise ValueError("Password required") + (pw, _) = hash_password(opts.password) + entry["password"] = pw + share_folder = os.path.join(FOLDERS, opts.folder) + share_config = os.path.join(CONFIGS, f"{opts.folder}.json") + if os.path.exists(share_config): + raise FileExistsError("Configuration already exists. Edit with `edit`") + + save_config(share_config, entry) + if not os.path.exists(share_folder): + os.mkdir(share_folder) + print(f"Added:\n{share_oneliner(load_config(opts.folder))}") + + +def share_remove(opts): + + share_folder = os.path.join(FOLDERS, opts.folder) + share_config = os.path.join(CONFIGS, f"{opts.folder}.json") + if not os.path.exists(share_config): + raise FileNotFoundError("Configuration doesn't exist.") + else: + print(f"Removing configuration {share_config}") + os.remove(share_config) + if os.path.exists(share_folder): + if len(os.listdir(share_folder)) > 0: + print(f"Not removing folder {share_folder}, contains data") + else: + print(f"Removing empty folder {share_folder}") + os.rmdir(share_folder) + + +def share_edit(opts): + + share_config = os.path.join(CONFIGS, f"{opts.folder}.json") + if not os.path.exists(share_config): + raise FileNotFoundError("Configuration doesn't exist.") + entry = load_config(opts.folder) + if opts.expires: + print(f"Updating expiry date: {opts.expires}") + if opts.expires == "never": + entry["expires"] = opts.expires + else: + entry["expires"] = ( + (datetime.now() + timedelta(days=float(opts.expires))) + .replace(microsecond=0) + .isoformat() + ) + if opts.password: + print("Updating password") + (pw, _) = hash_password(opts.password) + entry["password"] = pw + + save_config(share_config, entry) + print(f"Added:\n{share_oneliner(load_config(opts.folder))}") + + +def session_get_database(): + return sqlite3.connect(DATABASE) + + +def session_create_database(): + args = tuple() + db = session_get_database() + cur = db.execute(SCHEMA, args) + db.commit() + cur.close() + + +def session_list(): + query = "SELECT folder, expire, token, ip FROM sessions" + args = tuple() + cur = session_get_database().execute(query, args) + print(f"{'Share'.ljust(15)} {'Expires'.ljust(19)} IP") + for session in cur.fetchall(): + d = datetime.fromtimestamp(session[1]) + print(f"{session[0].ljust(15)} {d} {session[3]}") + cur.close() + + +def session_remove(): + query = "DELETE FROM sessions" + args = tuple() + db = session_get_database() + cur = db.execute(query, args) + db.commit() + cur.close() + + +def session_clean(): + query = "DELETE FROM sessions WHERE expire < ?" + args = (int(time.time()),) + db = session_get_database() + cur = db.execute(query, args) + db.commit() + cur.close() + + +def hash_password(pw): + pw = pw.encode("utf-8") + salt = bcrypt.gensalt() + hashed = bcrypt.hashpw(pw, salt) + return hashed.decode(), salt.decode() + + +if __name__ == "__main__": + manager() diff --git a/docker-pwss/code/start_daemon b/docker-pwss/code/start_daemon new file mode 100755 index 0000000..527269c --- /dev/null +++ b/docker-pwss/code/start_daemon @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +mkdir -p /data/static /data/configs +cd $( dirname $( readlink -f "$0" ) ) +. /venv/bin/activate +cat /venv/freeze.txt +supervisord \ + -c supervisord.conf \ + -n \ + -l /tmp/supervisord.log \ + -j /tmp/supervisord.pid + diff --git a/docker-pwss/code/supervisord.conf b/docker-pwss/code/supervisord.conf new file mode 100644 index 0000000..318ee2a --- /dev/null +++ b/docker-pwss/code/supervisord.conf @@ -0,0 +1,38 @@ +[supervisord] + +[program:maintain] +command=bash maintain +directory=/code +user=user +redirect_stderr=true +stdout_logfile=/proc/1/fd/1 +stderr_logfile=/proc/1/fd/1 +stdout_maxbytes=0 +stderr_maxbytes=0 +stdout_logfile_maxbytes = 0 +stderr_logfile_maxbytes = 0 + +[program:serve] +command=bash serve +directory=/code +user=user +redirect_stderr=true +stdout_logfile=/proc/1/fd/1 +stderr_logfile=/proc/1/fd/1 +stdout_maxbytes=0 +stderr_maxbytes=0 +stdout_logfile_maxbytes = 0 +stderr_logfile_maxbytes = 0 + +[program:memcached] +command=/usr/bin/memcached start -u memcached +directory=/ +user=user +redirect_stderr=true +stdout_logfile=/proc/1/fd/1 +stderr_logfile=/proc/1/fd/1 +stdout_maxbytes=0 +stderr_maxbytes=0 +stdout_logfile_maxbytes = 0 +stderr_logfile_maxbytes = 0 + diff --git a/docker-pwss/code/templates/index.html b/docker-pwss/code/templates/index.html new file mode 100644 index 0000000..8bb429d --- /dev/null +++ b/docker-pwss/code/templates/index.html @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + diff --git a/docker-pwss/code/templates/layout.html b/docker-pwss/code/templates/layout.html new file mode 100644 index 0000000..a2f3b89 --- /dev/null +++ b/docker-pwss/code/templates/layout.html @@ -0,0 +1,196 @@ + + + + + + + + + + + + + {% block body %}{% endblock %} + + + diff --git a/docker-pwss/code/templates/login.html b/docker-pwss/code/templates/login.html new file mode 100644 index 0000000..ed134f9 --- /dev/null +++ b/docker-pwss/code/templates/login.html @@ -0,0 +1,52 @@ +{% extends "layout.html" %} +{% block body %} +
+ {% if sessions %} +
+ + Open sessions:
+ {% for session in sessions %} + {{ session[0] }}/ +
    +
  • Session expires: {{ session[1] }} minutes
    • +
  • Share expires: {{ session[2] }} days
    • +
+ {% endfor %} +
+
+ {% endif %} +
+

Login to a share:

+
+ +
+

Visit Logout to log out all sessions.

+
+ +
+{% endblock %} diff --git a/docker-pwss/code/templates/ratelimit.html b/docker-pwss/code/templates/ratelimit.html new file mode 100644 index 0000000..2f04c79 --- /dev/null +++ b/docker-pwss/code/templates/ratelimit.html @@ -0,0 +1,55 @@ + + + + + + + + + + + + Ratelimit exceeded: {{ description }} + + + + diff --git a/docker-pwss/code/utils.py b/docker-pwss/code/utils.py new file mode 100644 index 0000000..8fb3015 --- /dev/null +++ b/docker-pwss/code/utils.py @@ -0,0 +1,131 @@ +from flask import request, current_app as app, g, session +from datetime import datetime +import bcrypt +import os +import json +import secrets +import sqlite3 +import time +import sys + +CONFIGS = os.getenv("CONFIG_FOLDER") + + +def check_password(config, pw): + pw = pw.encode("utf-8") + return bcrypt.checkpw(pw, config["password"].encode("utf-8")) + + +def check_auth(path): + folder = path.split(os.sep)[0] + if not f"auth/{folder}" in session: + return False + return has_session(folder) + + +def authenticate(config, password): + """Return success of authentication""" + if not "name" in config: + return False + folder = config["name"] + if config["expires"] == "never": + expiration = time.time() + app.config["SESSION_EXPIRY"] + else: + expiration = datetime.fromisoformat(config["expires"]).timestamp() + if f"auth/{folder}" in session: + session.pop(f"auth/{folder}") + if expiration > time.time(): + if "password" in config and check_password(config, password): + set_session(folder, max_expiration=expiration) + return True + return False + + +def get_db(): + db = getattr(g, "_database", None) + if db is None: + db = g._database = sqlite3.connect(app.config["DATABASE"]) + return db + + +def get_valid_sessions(): + query = "SELECT folder, expire, token FROM sessions WHERE expire > ? AND ip = ?" + args = (int(time.time()), get_ip()) + # TODO, validate that sessions have the token correct too. + tokens = [session[key] for key in session if key.startswith("auth/")] + try: + cur = get_db().execute(query, args) + valid_sessions = [ + ( + row[0], + int((row[1] - time.time()) / 60), + read_config(row[0]).get("days_left", "end of"), + ) + for row in cur.fetchall() + if row[2] in tokens + ] + cur.close() + return valid_sessions + except Exception as e: + print(e, file=sys.stderr) + return [] + + +def has_session(folder): + query = "SELECT count(token) FROM sessions WHERE folder = ? AND expire > ? AND token = ? AND ip = ?" + args = (folder, int(time.time()), session[f"auth/{folder}"], get_ip()) + try: + cur = get_db().execute(query, args) + is_valid = cur.fetchall()[0][0] > 0 + cur.close() + except Exception as e: + print(e, file=sys.stderr) + return False + return is_valid + + +def set_session(folder, max_expiration=None): + token = secrets.token_hex(16) + query = "INSERT INTO sessions (folder, expire, token, ip) VALUES (?,?,?,?)" + expiry_time = int(app.config["SESSION_EXPIRY"] + time.time()) + if max_expiration: + expiry_time = int(min(max_expiration, expiry_time)) + args = (folder, expiry_time, token, get_ip()) + db = get_db() + cur = db.execute(query, args) + cur.close() + db.commit() + session[f"auth/{folder}"] = token + + +def get_ip(): + + ip = request.environ.get( + "HTTP_X_FORWARDED_FOR", request.remote_addr or "127.0.0.1" + ) + ip = ip.split(",")[0].strip() + return ip + + +def read_config(path): + try: + rootdir = path.split(os.sep)[0] + config_file = os.path.join(CONFIGS, f"{rootdir}.json") + with open(config_file, "rt") as fp: + config = json.load(fp) + config["name"] = rootdir + try: + config["days_left"] = round( + ( + datetime.fromisoformat(config["expires"]).timestamp() + - time.time() + ) + / 86400, + 1, + ) + except ValueError: + config["days_left"] = "end of" + + return config + except (FileNotFoundError, AttributeError): + return {} diff --git a/env-example b/env-example new file mode 100644 index 0000000..a0fb837 --- /dev/null +++ b/env-example @@ -0,0 +1,10 @@ +EXPOSE=8088 +UUID=1000 +SESSION_EXPIRY=1800 +SECRET_KEY=2f6aa45dfcfc37a50537f0b05af6452c +CONFIG_FOLDER=/data/configs +STATIC_FOLDER=/data/static +DATABASE=/dev/shm/serve.db +WORKERS=6 +TZ=Europe/Helsinki +LIMITER_SHARE=200 per 5 seconds diff --git a/manage b/manage new file mode 100755 index 0000000..c99b81b --- /dev/null +++ b/manage @@ -0,0 +1,3 @@ +#!/bin/bash + +docker-compose exec pwss bash /code/manager "$@" diff --git a/service-down b/service-down new file mode 100755 index 0000000..ecaa80f --- /dev/null +++ b/service-down @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +docker-compose down diff --git a/service-up b/service-up new file mode 100755 index 0000000..dc4ce15 --- /dev/null +++ b/service-up @@ -0,0 +1,6 @@ +#!/bin/bash +set -e +mkdir -p data +docker-compose build +docker-compose up -d -t 0 +docker-compose logs -f