moonq/flees
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support for autoremoving X days old files

Browse Source
This commit is contained in:
Ville Rantanen
2018-11-17 23:24:32 +02:00
parent 70420bd035
commit 375628255d
5 changed files with 164 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
code/app.py
View File

@@ -304,6 +304,8 @@ def file_list(name, token):
return share
files = []
for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']):
if file_autoremove(file, share, notify):
continue
files.append(path2url(file))
files.append("")
return "\n".join(files), 200
@@ -379,6 +381,8 @@ def file_ls(name, token):
files = []
maxlen = 4
for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']):
if file_autoremove(file, share, notify):
continue
files.append(file)
maxlen = max(maxlen, len(file))
details = []
@@ -437,6 +441,8 @@ def list_view(name, token = None):
files = []
for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']):
if file_autoremove(file, share, notify):
continue
status = file_stat(share['path'],file)
status.update({
'token': get_direct_token(share, file),
@@ -500,6 +506,7 @@ def download_direct(name,token,filename):
"operation": "unauthorized_direct_download"
})
return 'Incorrect token', 403
file_autoremove(filename, share, notify)
file_path = os.path.join(share['path'], filename)
if not os.path.exists(file_path):
return 'No such file', 404
@@ -767,6 +774,7 @@ def download_file(name, filename, token = None):
return share
if not is_path_safe(filename):
return "Incorrect path", 403
file_autoremove(filename, share, notify)
file_path = os.path.join(share['path'], filename)
if not os.path.exists(file_path):
return 'No such file, '+file_path, 404
@@ -896,6 +904,8 @@ def zip_share(share):
)
zf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
for file in iter_folder_files(share['path'], version_folder = app.config['VERSION_FOLDER']):
if file_autoremove(file, share, notify):
continue
fp = os.path.join(share['path'], file)
if os.path.isdir(fp):
continue

1
code/docker-requirements.txt
View File

@@ -3,3 +3,4 @@ gunicorn
pycrypto
requests
python-magic
python-dateutil

119
code/flees-manager.py
View File

@@ -20,7 +20,7 @@ def get_root_path(opts):
def list_shares(shares,opts):
table = []
header = ('Name', 'Path','Public','Password','Token','Upload','Overwrite','Direct','Expire','Recipient','Description')
header = ('Name', 'Path','Public','Password','Token','Upload','Overwrite','Direct','Expire','AutoRemove','Recipient','Description')
short_header = ('Name', 'Path','Pub','Pwd','Up','Drct','Exp')
for share in shares:
public = get_or_none('public',share, False)
@@ -30,6 +30,11 @@ def list_shares(shares,opts):
tokens = len(share['tokens']) > 0
upload = get_or_none('upload',share, False)
overwrite = get_or_none('overwrite',share, True)
autoremove = get_or_none('autoremove', share, 0)
if autoremove > 0:
autoremove = "%d d"%( autoremove, )
else:
autoremove = "-"
direct = get_or_none('direct_links',share, False) if password else False
expire = get_or_none('expire',share, "-")
if not opts.verbose:
@@ -45,6 +50,7 @@ def list_shares(shares,opts):
overwrite,
direct,
expire,
autoremove,
get_or_none('recipient', share, "")[0:20],
description
))
@@ -61,7 +67,6 @@ def list_shares(shares,opts):
)
def list_folders(shares,config):
if 'data_folder' not in config:
print("data_folder not defined in config")
@@ -174,6 +179,59 @@ def list_versions(shares, config, opts):
print(tabulate(table, headers = header))
def list_autoremove(shares, config, opts):
if 'data_folder' not in config:
print("data_folder not defined in config")
sys.exit(1)
data_folder = os.path.join(config['__root_path__'], config['data_folder'])
table = []
header = ['Path', 'Size', 'Age', 'ToDelete']
for share in shares:
if opts.name:
if share['name'] != opts.name:
continue
share_folder = os.path.join(
data_folder,
share['path'],
)
autoremove = get_or_none('autoremove', share, 0)
if autoremove == 0:
autoremove = False
del header[-1]
for filename in iter_folder_files(share_folder):
full_path = os.path.join(share_folder, filename)
if os.path.isdir(full_path):
continue
size = os.path.getsize(full_path)
size_str = file_size_human(
size,
HTML = False
)
age, age_str = file_age(full_path)
to_delete = age.days >= autoremove
if not autoremove:
to_delete = False
del_str = "Yes" if to_delete else "No"
if opts.delete:
if to_delete:
del_str = "Deleted"
os.remove(full_path)
row_data = [
os.path.join(
share['path'],
filename
),
size_str,
age_str,
del_str
]
if not autoremove:
del row_data[-1]
table.append(row_data)
table.sort(key = lambda x: x[0])
print(tabulate(table, headers = header))
def add_share(shares, config, opts):
# Make name and path safe:
@@ -194,7 +252,8 @@ def add_share(shares, config, opts):
'overwrite': opts.overwrite,
'direct_links': opts.direct,
'description': opts.description,
'recipient': opts.recipient
'recipient': opts.recipient,
'autoremove': opts.autoremove
}
if opts.password:
@@ -280,6 +339,8 @@ def modify_share(shares, config, opts):
share['description'] = opts.description
if opts.recipient != None:
share['recipient'] = opts.recipient
if opts.autoremove != None:
share['autoremove'] = opts.autoremove
# REMOVE password
if opts.password == "":
if 'pass_plain' in share:
@@ -641,6 +702,25 @@ def parse_options():
help = "Do not actually delete files.")
parser_versions.add_argument('-n','--name', action="store", dest="name", required = False, default = None,
help = "Show / Delete only this share versions. If omitted, applies to all shares")
## list autoremove
parser_autoremove = subparsers.add_parser(
'ages',
help = "List the file ages in shares, and their disk usage"
)
parser_autoremove.add_argument(
'--delete',
action = "store_true",
dest = "delete",
default = False,
help = "Delete files older than share autoremove value."
)
parser_autoremove.add_argument(
'-n','--name',
action = "store",
dest = "name",
required = True,
default = None,
help = "Share name to show / delete from.")
## Show
parser_show = subparsers.add_parser('show', help = "Show share")
parser_show.add_argument('-P', action="store_true", dest="show_password", default = False,
@@ -654,9 +734,18 @@ def parse_options():
)
## Add
parser_add = subparsers.add_parser('add', help = "Add a share")
parser_add.add_argument('-n','--name', action="store", dest="name", required = True)
parser_add.add_argument('-p','--path', action="store", dest="path", required = True,
help= "path relative to data folder"
parser_add.add_argument(
'-n','--name',
action = "store",
dest = "name",
required = True
)
parser_add.add_argument(
'-p','--path',
action = "store",
dest = "path",
required = True,
help = "path relative to data folder"
)
parser_add.add_argument('-D','--description', action="store", dest="description", default = "",
help= "Describe the contents"
@@ -674,6 +763,14 @@ def parse_options():
parser_add.add_argument('-e','--expire', action="store", dest="expire", default = False,
help = "expire date in format '%%Y-%%m-%%d %%H:%%M' ex. '2018-12-24 21:00'"
)
parser_add.add_argument(
'--rm','--autoremove',
action = "store",
dest = "autoremove",
default = 0,
type = int,
help = "Remove files older than N days. 0 disables the feature."
)
parser_add.add_argument('-r','--recipient', action="store", dest="recipient", default = "",
help= "Recipient for notifications (if enabled)"
)
@@ -702,6 +799,14 @@ def parse_options():
parser_modify.add_argument('-e','--expire', action="store", dest="expire", default = False,
help = "expire date in format '%%Y-%%m-%%d %%H:%%M' ex. '2018-12-24 21:00'. Set as empty string to remove expiration."
)
parser_modify.add_argument(
'--rm','--autoremove',
action = "store",
dest = "autoremove",
default = None,
type = int,
help = "Remove files older than N days. 0 disables the feature."
)
parser_modify.add_argument('-r','--recipient', action="store", dest="recipient", default = None,
help= "Recipient for notifications (if enabled)"
)
@@ -758,6 +863,8 @@ if __name__ == "__main__":
list_shares(shares,opts)
if opts.subparser_name == 'versions':
list_versions(shares,config,opts)
if opts.subparser_name == 'ages':
list_autoremove(shares,config,opts)
elif opts.subparser_name == 'folders':
list_folders(shares,config)
elif opts.subparser_name == 'show':

1
code/manager-requirements.txt
View File

@@ -1,3 +1,4 @@
tabulate
#pycrypto
requests
python-dateutil

39
code/utils/utils.py
View File

@@ -1,5 +1,6 @@
import os
from datetime import datetime
from dateutil.relativedelta import relativedelta
from flask import current_app as app
import requests
import re
@@ -92,6 +93,44 @@ def download_url(url, filename):
return (True, ("OK", 200 ))
def file_autoremove(path, share, notifier = None):
autoremove = get_or_none('autoremove', share, 0)
if autoremove == 0:
return
full_path = os.path.join(
share['path'],
path
)
age, age_str = file_age(full_path)
if age.days >= autoremove:
os.remove(full_path)
if notifier != None:
notifier({
"recipient": get_or_none('recipient', share),
"share": share['name'],
"filename": full_path,
"file_age": age_str,
"operation": "autoremove"
})
return True
return False
def file_age(path):
now = datetime.now()
then = datetime.fromtimestamp(
os.stat(path).st_mtime
)
diff = now - then
rdiff = relativedelta(now, then)
return diff, "%dM %02dD %02d:%02dH" % (
rdiff.years * 12 + rdiff.months,
rdiff.days,
rdiff.hours,
rdiff.minutes
)
def file_date_human(num):
return datetime.fromtimestamp(
num