support for autoremoving X days old files

code/app.py

@@ -304,6 +304,8 @@ def file_list(name, token):
         return share
     files = []
     for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']):
+        if file_autoremove(file, share, notify):
+            continue
         files.append(path2url(file))
     files.append("")
     return  "\n".join(files), 200
@@ -379,6 +381,8 @@ def file_ls(name, token):
     files = []
     maxlen = 4
     for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']):
+        if file_autoremove(file, share, notify):
+            continue
         files.append(file)
         maxlen = max(maxlen, len(file))
     details = []
@@ -437,6 +441,8 @@ def list_view(name, token = None):
 
     files = []
     for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']):
+        if file_autoremove(file, share, notify):
+            continue
         status = file_stat(share['path'],file)
         status.update({
             'token': get_direct_token(share, file),
@@ -500,6 +506,7 @@ def download_direct(name,token,filename):
             "operation": "unauthorized_direct_download"
         })
         return 'Incorrect token', 403
+    file_autoremove(filename, share, notify)
     file_path = os.path.join(share['path'], filename)
     if not os.path.exists(file_path):
         return 'No such file', 404
@@ -767,6 +774,7 @@ def download_file(name, filename, token = None):
         return share
     if not is_path_safe(filename):
         return "Incorrect path", 403
+    file_autoremove(filename, share, notify)
     file_path = os.path.join(share['path'], filename)
     if not os.path.exists(file_path):
         return 'No such file, '+file_path, 404
@@ -896,6 +904,8 @@ def zip_share(share):
     )
     zf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
     for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']):
+        if file_autoremove(file, share, notify):
+            continue
         fp = os.path.join(share['path'], file)
         if os.path.isdir(fp):
             continue

code/docker-requirements.txt

@@ -3,3 +3,4 @@ gunicorn
 pycrypto
 requests
 python-magic
+python-dateutil

code/flees-manager.py

@@ -20,7 +20,7 @@ def get_root_path(opts):
 
 def list_shares(shares,opts):
     table = []
-    header = ('Name', 'Path','Public','Password','Token','Upload','Overwrite','Direct','Expire','Recipient','Description')
+    header = ('Name', 'Path','Public','Password','Token','Upload','Overwrite','Direct','Expire','AutoRemove','Recipient','Description')
     short_header = ('Name', 'Path','Pub','Pwd','Up','Drct','Exp')
     for share in shares:
         public = get_or_none('public',share, False)
@@ -30,6 +30,11 @@ def list_shares(shares,opts):
             tokens = len(share['tokens']) > 0
         upload = get_or_none('upload',share, False)
         overwrite = get_or_none('overwrite',share, True)
+        autoremove = get_or_none('autoremove', share, 0)
+        if autoremove > 0:
+            autoremove = "%d d"%( autoremove, )
+        else:
+            autoremove = "-"
         direct = get_or_none('direct_links',share, False) if password else False
         expire = get_or_none('expire',share, "-")
         if not opts.verbose:
@@ -45,6 +50,7 @@ def list_shares(shares,opts):
             overwrite,
             direct,
             expire,
+            autoremove,
             get_or_none('recipient', share, "")[0:20],
             description
         ))
@@ -61,7 +67,6 @@ def list_shares(shares,opts):
     )
 
 
-
 def list_folders(shares,config):
     if 'data_folder' not in config:
         print("data_folder not defined in config")
@@ -174,6 +179,59 @@ def list_versions(shares, config, opts):
     print(tabulate(table, headers = header))
 
 
+def list_autoremove(shares, config, opts):
+    if 'data_folder' not in config:
+        print("data_folder not defined in config")
+        sys.exit(1)
+    data_folder = os.path.join(config['__root_path__'], config['data_folder'])
+    table = []
+    header = ['Path', 'Size', 'Age', 'ToDelete']
+    for share in shares:
+        if opts.name:
+            if share['name'] != opts.name:
+                continue
+        share_folder = os.path.join(
+            data_folder,
+            share['path'],
+        )
+        autoremove = get_or_none('autoremove', share, 0)
+        if autoremove == 0:
+            autoremove = False
+            del header[-1]
+        for filename in iter_folder_files(share_folder):
+            full_path = os.path.join(share_folder, filename)
+            if os.path.isdir(full_path):
+                continue
+            size = os.path.getsize(full_path)
+            size_str = file_size_human(
+                size,
+                HTML = False
+            )
+            age, age_str = file_age(full_path)
+            to_delete = age.days >= autoremove
+            if not autoremove:
+                to_delete = False
+            del_str = "Yes" if to_delete else "No"
+            if opts.delete:
+                if to_delete:
+                    del_str = "Deleted"
+                    os.remove(full_path)
+            row_data = [
+                os.path.join(
+                    share['path'],
+                    filename
+                ),
+                size_str,
+                age_str,
+                del_str
+            ]
+            if not autoremove:
+                del row_data[-1]
+            table.append(row_data)
+    table.sort(key = lambda x: x[0])
+    print(tabulate(table, headers = header))
+
+
 def add_share(shares, config, opts):
 
     # Make name and path safe:
@@ -194,7 +252,8 @@ def add_share(shares, config, opts):
         'overwrite': opts.overwrite,
         'direct_links': opts.direct,
         'description': opts.description,
-        'recipient': opts.recipient
+        'recipient': opts.recipient,
+        'autoremove': opts.autoremove
     }
 
     if opts.password:
@@ -280,6 +339,8 @@ def modify_share(shares, config, opts):
         share['description'] = opts.description
     if opts.recipient != None:
         share['recipient'] = opts.recipient
+    if opts.autoremove != None:
+        share['autoremove'] = opts.autoremove
     # REMOVE password
     if opts.password == "":
         if 'pass_plain' in share:
@@ -641,6 +702,25 @@ def parse_options():
         help = "Do not actually delete files.")
     parser_versions.add_argument('-n','--name', action="store", dest="name", required = False, default = None,
         help = "Show / Delete only this share versions. If omitted, applies to all shares")
+    ## list autoremove
+    parser_autoremove = subparsers.add_parser(
+        'ages',
+        help = "List the file ages in shares, and their disk usage"
+    )
+    parser_autoremove.add_argument(
+        '--delete',
+        action = "store_true",
+        dest = "delete",
+        default = False,
+        help = "Delete files older than share autoremove value."
+    )
+    parser_autoremove.add_argument(
+        '-n','--name',
+        action = "store",
+        dest = "name",
+        required = True,
+        default = None,
+        help = "Share name to show / delete from.")
     ## Show
     parser_show = subparsers.add_parser('show', help = "Show share")
     parser_show.add_argument('-P', action="store_true", dest="show_password", default = False,
@@ -654,8 +734,17 @@ def parse_options():
     )
     ## Add
     parser_add = subparsers.add_parser('add', help = "Add a share")
-    parser_add.add_argument('-n','--name', action="store", dest="name", required = True)
+    parser_add.add_argument(
-    parser_add.add_argument('-p','--path', action="store", dest="path", required = True,
+        '-n','--name',
+        action = "store",
+        dest = "name",
+        required = True
+    )
+    parser_add.add_argument(
+        '-p','--path',
+        action = "store",
+        dest = "path",
+        required = True,
         help = "path relative to data folder"
     )
     parser_add.add_argument('-D','--description', action="store", dest="description", default = "",
@@ -674,6 +763,14 @@ def parse_options():
     parser_add.add_argument('-e','--expire', action="store", dest="expire", default = False,
         help  = "expire date in format '%%Y-%%m-%%d %%H:%%M' ex. '2018-12-24 21:00'"
     )
+    parser_add.add_argument(
+        '--rm','--autoremove',
+        action = "store",
+        dest = "autoremove",
+        default = 0,
+        type = int,
+        help = "Remove files older than N days. 0 disables the feature."
+    )
     parser_add.add_argument('-r','--recipient', action="store", dest="recipient", default = "",
         help= "Recipient for notifications (if enabled)"
     )
@@ -702,6 +799,14 @@ def parse_options():
     parser_modify.add_argument('-e','--expire', action="store", dest="expire", default = False,
         help  = "expire date in format '%%Y-%%m-%%d %%H:%%M' ex. '2018-12-24 21:00'. Set as empty string to remove expiration."
     )
+    parser_modify.add_argument(
+        '--rm','--autoremove',
+        action = "store",
+        dest = "autoremove",
+        default = None,
+        type = int,
+        help = "Remove files older than N days. 0 disables the feature."
+    )
     parser_modify.add_argument('-r','--recipient', action="store", dest="recipient", default = None,
         help= "Recipient for notifications (if enabled)"
     )
@@ -758,6 +863,8 @@ if __name__ == "__main__":
         list_shares(shares,opts)
     if opts.subparser_name == 'versions':
         list_versions(shares,config,opts)
+    if opts.subparser_name == 'ages':
+        list_autoremove(shares,config,opts)
     elif opts.subparser_name == 'folders':
         list_folders(shares,config)
     elif opts.subparser_name == 'show':

code/manager-requirements.txt

@@ -1,3 +1,4 @@
 tabulate
 #pycrypto
 requests
+python-dateutil

code/utils/utils.py

@@ -1,5 +1,6 @@
 import os
 from datetime import datetime
+from dateutil.relativedelta import relativedelta
 from flask import current_app as app
 import requests
 import re
@@ -92,6 +93,44 @@ def download_url(url, filename):
     return (True, ("OK", 200 ))
 
 
+def file_autoremove(path, share, notifier = None):
+    autoremove = get_or_none('autoremove', share, 0)
+    if autoremove == 0:
+        return
+    full_path = os.path.join(
+        share['path'],
+        path
+    )
+    age, age_str = file_age(full_path)
+    if age.days >= autoremove:
+        os.remove(full_path)
+        if notifier != None:
+            notifier({
+                "recipient": get_or_none('recipient', share),
+                "share": share['name'],
+                "filename": full_path,
+                "file_age": age_str,
+                "operation": "autoremove"
+            })
+        return True
+    return False
+
+
+def file_age(path):
+    now = datetime.now()
+    then = datetime.fromtimestamp(
+        os.stat(path).st_mtime
+    )
+    diff = now - then
+    rdiff = relativedelta(now, then)
+    return diff, "%dM %02dD %02d:%02dH" % (
+        rdiff.years * 12 + rdiff.months,
+        rdiff.days,
+        rdiff.hours,
+        rdiff.minutes
+    )
+
+
 def file_date_human(num):
     return datetime.fromtimestamp(
         num