diff --git a/code/app.py b/code/app.py index e5087c5..0f833f1 100644 --- a/code/app.py +++ b/code/app.py @@ -304,6 +304,8 @@ def file_list(name, token): return share files = [] for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']): + if file_autoremove(file, share, notify): + continue files.append(path2url(file)) files.append("") return "\n".join(files), 200 @@ -379,6 +381,8 @@ def file_ls(name, token): files = [] maxlen = 4 for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']): + if file_autoremove(file, share, notify): + continue files.append(file) maxlen = max(maxlen, len(file)) details = [] @@ -437,6 +441,8 @@ def list_view(name, token = None): files = [] for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']): + if file_autoremove(file, share, notify): + continue status = file_stat(share['path'],file) status.update({ 'token': get_direct_token(share, file), @@ -500,6 +506,7 @@ def download_direct(name,token,filename): "operation": "unauthorized_direct_download" }) return 'Incorrect token', 403 + file_autoremove(filename, share, notify) file_path = os.path.join(share['path'], filename) if not os.path.exists(file_path): return 'No such file', 404 @@ -767,6 +774,7 @@ def download_file(name, filename, token = None): return share if not is_path_safe(filename): return "Incorrect path", 403 + file_autoremove(filename, share, notify) file_path = os.path.join(share['path'], filename) if not os.path.exists(file_path): return 'No such file, '+file_path, 404 @@ -896,6 +904,8 @@ def zip_share(share): ) zf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']): + if file_autoremove(file, share, notify): + continue fp = os.path.join(share['path'], file) if os.path.isdir(fp): continue diff --git a/code/docker-requirements.txt b/code/docker-requirements.txt index 3f13af6..32fa877 100644 --- a/code/docker-requirements.txt +++ b/code/docker-requirements.txt @@ -3,3 +3,4 @@ gunicorn pycrypto requests python-magic +python-dateutil diff --git a/code/flees-manager.py b/code/flees-manager.py index 6a8b7cb..914c323 100755 --- a/code/flees-manager.py +++ b/code/flees-manager.py @@ -20,7 +20,7 @@ def get_root_path(opts): def list_shares(shares,opts): table = [] - header = ('Name', 'Path','Public','Password','Token','Upload','Overwrite','Direct','Expire','Recipient','Description') + header = ('Name', 'Path','Public','Password','Token','Upload','Overwrite','Direct','Expire','AutoRemove','Recipient','Description') short_header = ('Name', 'Path','Pub','Pwd','Up','Drct','Exp') for share in shares: public = get_or_none('public',share, False) @@ -30,6 +30,11 @@ def list_shares(shares,opts): tokens = len(share['tokens']) > 0 upload = get_or_none('upload',share, False) overwrite = get_or_none('overwrite',share, True) + autoremove = get_or_none('autoremove', share, 0) + if autoremove > 0: + autoremove = "%d d"%( autoremove, ) + else: + autoremove = "-" direct = get_or_none('direct_links',share, False) if password else False expire = get_or_none('expire',share, "-") if not opts.verbose: @@ -45,6 +50,7 @@ def list_shares(shares,opts): overwrite, direct, expire, + autoremove, get_or_none('recipient', share, "")[0:20], description )) @@ -61,7 +67,6 @@ def list_shares(shares,opts): ) - def list_folders(shares,config): if 'data_folder' not in config: print("data_folder not defined in config") @@ -174,6 +179,59 @@ def list_versions(shares, config, opts): print(tabulate(table, headers = header)) +def list_autoremove(shares, config, opts): + if 'data_folder' not in config: + print("data_folder not defined in config") + sys.exit(1) + data_folder = os.path.join(config['__root_path__'], config['data_folder']) + table = [] + header = ['Path', 'Size', 'Age', 'ToDelete'] + for share in shares: + if opts.name: + if share['name'] != opts.name: + continue + share_folder = os.path.join( + data_folder, + share['path'], + ) + autoremove = get_or_none('autoremove', share, 0) + if autoremove == 0: + autoremove = False + del header[-1] + for filename in iter_folder_files(share_folder): + full_path = os.path.join(share_folder, filename) + if os.path.isdir(full_path): + continue + size = os.path.getsize(full_path) + size_str = file_size_human( + size, + HTML = False + ) + age, age_str = file_age(full_path) + to_delete = age.days >= autoremove + if not autoremove: + to_delete = False + del_str = "Yes" if to_delete else "No" + if opts.delete: + if to_delete: + del_str = "Deleted" + os.remove(full_path) + row_data = [ + os.path.join( + share['path'], + filename + ), + size_str, + age_str, + del_str + ] + if not autoremove: + del row_data[-1] + table.append(row_data) + table.sort(key = lambda x: x[0]) + print(tabulate(table, headers = header)) + + def add_share(shares, config, opts): # Make name and path safe: @@ -194,7 +252,8 @@ def add_share(shares, config, opts): 'overwrite': opts.overwrite, 'direct_links': opts.direct, 'description': opts.description, - 'recipient': opts.recipient + 'recipient': opts.recipient, + 'autoremove': opts.autoremove } if opts.password: @@ -280,6 +339,8 @@ def modify_share(shares, config, opts): share['description'] = opts.description if opts.recipient != None: share['recipient'] = opts.recipient + if opts.autoremove != None: + share['autoremove'] = opts.autoremove # REMOVE password if opts.password == "": if 'pass_plain' in share: @@ -641,6 +702,25 @@ def parse_options(): help = "Do not actually delete files.") parser_versions.add_argument('-n','--name', action="store", dest="name", required = False, default = None, help = "Show / Delete only this share versions. If omitted, applies to all shares") + ## list autoremove + parser_autoremove = subparsers.add_parser( + 'ages', + help = "List the file ages in shares, and their disk usage" + ) + parser_autoremove.add_argument( + '--delete', + action = "store_true", + dest = "delete", + default = False, + help = "Delete files older than share autoremove value." + ) + parser_autoremove.add_argument( + '-n','--name', + action = "store", + dest = "name", + required = True, + default = None, + help = "Share name to show / delete from.") ## Show parser_show = subparsers.add_parser('show', help = "Show share") parser_show.add_argument('-P', action="store_true", dest="show_password", default = False, @@ -654,9 +734,18 @@ def parse_options(): ) ## Add parser_add = subparsers.add_parser('add', help = "Add a share") - parser_add.add_argument('-n','--name', action="store", dest="name", required = True) - parser_add.add_argument('-p','--path', action="store", dest="path", required = True, - help= "path relative to data folder" + parser_add.add_argument( + '-n','--name', + action = "store", + dest = "name", + required = True + ) + parser_add.add_argument( + '-p','--path', + action = "store", + dest = "path", + required = True, + help = "path relative to data folder" ) parser_add.add_argument('-D','--description', action="store", dest="description", default = "", help= "Describe the contents" @@ -674,6 +763,14 @@ def parse_options(): parser_add.add_argument('-e','--expire', action="store", dest="expire", default = False, help = "expire date in format '%%Y-%%m-%%d %%H:%%M' ex. '2018-12-24 21:00'" ) + parser_add.add_argument( + '--rm','--autoremove', + action = "store", + dest = "autoremove", + default = 0, + type = int, + help = "Remove files older than N days. 0 disables the feature." + ) parser_add.add_argument('-r','--recipient', action="store", dest="recipient", default = "", help= "Recipient for notifications (if enabled)" ) @@ -702,6 +799,14 @@ def parse_options(): parser_modify.add_argument('-e','--expire', action="store", dest="expire", default = False, help = "expire date in format '%%Y-%%m-%%d %%H:%%M' ex. '2018-12-24 21:00'. Set as empty string to remove expiration." ) + parser_modify.add_argument( + '--rm','--autoremove', + action = "store", + dest = "autoremove", + default = None, + type = int, + help = "Remove files older than N days. 0 disables the feature." + ) parser_modify.add_argument('-r','--recipient', action="store", dest="recipient", default = None, help= "Recipient for notifications (if enabled)" ) @@ -758,6 +863,8 @@ if __name__ == "__main__": list_shares(shares,opts) if opts.subparser_name == 'versions': list_versions(shares,config,opts) + if opts.subparser_name == 'ages': + list_autoremove(shares,config,opts) elif opts.subparser_name == 'folders': list_folders(shares,config) elif opts.subparser_name == 'show': diff --git a/code/manager-requirements.txt b/code/manager-requirements.txt index 46f457c..7a8f544 100644 --- a/code/manager-requirements.txt +++ b/code/manager-requirements.txt @@ -1,3 +1,4 @@ tabulate #pycrypto requests +python-dateutil diff --git a/code/utils/utils.py b/code/utils/utils.py index 23a54e0..4fbc282 100644 --- a/code/utils/utils.py +++ b/code/utils/utils.py @@ -1,5 +1,6 @@ import os from datetime import datetime +from dateutil.relativedelta import relativedelta from flask import current_app as app import requests import re @@ -92,6 +93,44 @@ def download_url(url, filename): return (True, ("OK", 200 )) +def file_autoremove(path, share, notifier = None): + autoremove = get_or_none('autoremove', share, 0) + if autoremove == 0: + return + full_path = os.path.join( + share['path'], + path + ) + age, age_str = file_age(full_path) + if age.days >= autoremove: + os.remove(full_path) + if notifier != None: + notifier({ + "recipient": get_or_none('recipient', share), + "share": share['name'], + "filename": full_path, + "file_age": age_str, + "operation": "autoremove" + }) + return True + return False + + +def file_age(path): + now = datetime.now() + then = datetime.fromtimestamp( + os.stat(path).st_mtime + ) + diff = now - then + rdiff = relativedelta(now, then) + return diff, "%dM %02dD %02d:%02dH" % ( + rdiff.years * 12 + rdiff.months, + rdiff.days, + rdiff.hours, + rdiff.minutes + ) + + def file_date_human(num): return datetime.fromtimestamp( num