import os from datetime import datetime from dateutil.relativedelta import relativedelta from flask import current_app as app import requests import re import json import stat try: import magic except ImportError: pass try: from werkzeug.utils import secure_filename except ImportError: pass try: from urllib.request import pathname2url from urllib.request import urlparse except ImportError: from urllib import pathname2url from urlparse import urlparse class Logger: def __init__(self, filename, uid = 0, gid = 0): self.filename = filename self.uid = uid self.gid = gid self.init() def info(self, message): self.write("INFO", str(message)) def init(self): self.info("Service started") self.set_rights() def set_rights(self): os.chown( self.filename, self.uid, self.gid ) st = os.stat(self.filename) if self.uid > 0: os.chmod( self.filename, st.st_mode | stat.S_IRUSR | stat.S_IWUSR ) if self.gid > 0: os.chmod( self.filename, st.st_mode | stat.S_IRGRP | stat.S_IWGRP ) def warning(self, message): self.write("WARNING", str(message)) def write(self, level, message): with open(self.filename, 'at') as fp: fp.write("%s\t%s\t%s\n"%( datetime.now().isoformat(), level, message )) fp.flush() def bool_short(var): if type(var) == bool: if var: return "Y" else: return "N" return var def download_url(url, filename): try: r = requests.get(url, stream=True) with open(filename, 'wb') as f: for chunk in r.iter_content(chunk_size=1024 * 1024): if chunk: # filter out keep-alive new chunks f.write(chunk) except requests.exceptions.RequestException as e: return (False, ("%s %s"%(e.code,e.reason), e.code)) return (True, ("OK", 200 )) def file_autoremove(path, share, notifier = None): autoremove = get_or_none('autoremove', share, 0) if autoremove == 0: return full_path = os.path.join( share['path'], path ) age, age_str = file_age(full_path) if age.days >= autoremove: os.remove(full_path) if notifier != None: notifier({ "recipient": get_or_none('recipient', share), "share": share['name'], "filename": full_path, "file_age": age_str, "operation": "autoremove" }) return True return False def file_age(path): now = datetime.now() then = datetime.fromtimestamp( os.stat(path).st_mtime ) diff = now - then rdiff = relativedelta(now, then) return diff, "%dM %02dD %02d:%02dH" % ( rdiff.years * 12 + rdiff.months, rdiff.days, rdiff.hours, rdiff.minutes ) def file_date_human(num): return datetime.fromtimestamp( num ).strftime(app.config['DATE_FORMAT']) def file_name_version(full_path): """ New name versioned with date of the file """ file_dir = os.path.dirname(full_path) file_name = os.path.basename(full_path) file_time = os.stat(full_path).st_mtime time_formatted = datetime.fromtimestamp( file_time ).strftime('%Y%m%d_%H%M%S') new_name = '%s.%s'%( time_formatted, file_name ) return new_name def version_date(full_path): """ Date of versioned file """ file_dir = os.path.dirname(full_path) file_name = os.path.basename(full_path) try: return datetime.strptime( file_name[0:15], '%Y%m%d_%H%M%S' ) except ValueError: return None def file_mime(filename): try: return magic.from_file(filename, mime = True) except NameError: # magic not imported return "NA" def file_stat(path, filename): full_path = os.path.join(path, filename) s = os.stat(full_path) return { 'size': file_size_MB(s.st_size), 'hsize': file_size_human(s.st_size, HTML = False), 'mtime': file_date_human(s.st_mtime), 'name': filename, 'url': path2url(filename), 'editable': (s.st_size < 65536 and filename.endswith(".txt")), 'mime': file_mime(full_path) } def file_size_human(num, HTML = True): space = ' ' if HTML else ' ' for x in [space + 'B', 'KB', 'MB', 'GB', 'TB']: if num < 1024.0: if x == space + 'B': return "%d%s%s" % (num, space, x) return "%3.1f%s%s" % (num, space, x) num /= 1024.0 def file_size_MB(num): return "{:,.2f}".format(num/(1024*1024)) def get_folder_size(path): total_size = 0 for dirpath, dirnames, filenames in os.walk(path): for f in filenames: fp = os.path.join(dirpath, f) total_size += os.path.getsize(fp) return total_size def get_or_none(key,d,none = None): if key in d: return d[key] else: return none def get_script_url(public_url, share, end_point, token = "[TOKEN]"): cmd = None doc = None if get_or_none("direct_links", share) and end_point == "download": end_point = "direct" url = "%s/script/%s/%s/%s"%( public_url, end_point, share['name'], token ) if end_point in ( "download", "direct"): cmd = 'curl -s %s | bash /dev/stdin [-f]'%( url, ) doc = 'Download all files in the share. -f to force overwrite existing files.' if end_point == "client": cmd = 'python <( curl -s %s )'%( url, ) doc = 'Console client to download and upload files.' if end_point == "upload_split": cmd = 'curl -s %s | python - [-s split_size_in_Mb] file_to_upload.ext [second.file.ext]'%( url, ) doc = 'Upload files to the share. -s to set splitting size.' if end_point == "flip": cmd = 'curl -s %s > flip && ./flip'%( url, ) doc = 'Use the share as a command line clipboard' return {'cmd': cmd, 'doc': doc} def is_path_safe(path): if path.startswith("."): return False if "/." in path: return False return True def is_valid_url(url, qualifying = None): min_attributes = ('scheme', 'netloc') qualifying = min_attributes if qualifying is None else qualifying token = urlparse(url) return all([getattr(token, qualifying_attr) for qualifying_attr in qualifying]) def iter_folder_files(path, recursive = True, version_folder = None): if recursive: for dirpath, dirnames, filenames in os.walk(path, topdown = False): path_list = dirpath.split(os.sep) if path_list[-1] == version_folder: continue relative_path = os.path.relpath(dirpath,path) dirnames.sort() if "/." in relative_path: continue if relative_path == ".": relative_path = "" for f in sorted(filenames): if f.startswith("."): continue fp = os.path.join(relative_path, f) yield fp else: for file in sorted(path): fp = os.path.join(path,file) if os.path.isdir(fp): continue if file.startswith("."): continue yield fp def path2url(path): return pathname2url(path) def read_config(app): # Read config from json config_values = json.load(open(os.getenv('FLEES_CONFIG'),'rt')) app.config['PUBLIC_URL'] = config_values['public_url'] app.config['SITE_NAME'] = config_values['site_name'] app.config['UPLOAD_FOLDER'] = config_values['data_folder'] app.config['SHARES_FILE'] = config_values['shares_file'] if 'log_file' in config_values: app.config['LOG_FILE'] = config_values['log_file'] else: app.config['LOG_FILE'] = os.path.join(app.config['UPLOAD_FOLDER'], 'flees.log') app.config['ZIP_FOLDER'] = config_values['zip_folder'] app.config['MAX_ZIP_SIZE'] = config_values['max_zip_size'] # megabytes app.config['DATE_FORMAT'] = config_values['date_format'] app.config['UID'] = config_values['uid'] app.config['GID'] = config_values['gid'] app.config['DEBUG'] = config_values['debug'] app.config['VERSION_FOLDER'] = config_values['version_folder'] app.config['SESSION_COOKIE_NAME'] = secure_filename(config_values['site_name']) app.config['LOGGER'] = Logger( app.config['LOG_FILE'], app.config['UID'], app.config['GID'] ) return config_values def safe_name(s): return safe_string(s, "-_") def safe_path(s): return safe_string(s, "-_/") def safe_string(s, valid, no_repeat = False): """ return a safe string, replace non alnum characters with _ . all characters in valid are considered valid. """ safe = "".join([c if c.isalnum() or c in valid else "_" for c in s]) if no_repeat: safe = re.sub(r'_+', '_', safe) return safe def set_rights(path): os.chown(path, app.config['UID'], app.config['GID']) st = os.stat(path) if app.config['UID'] > 0: os.chmod(path, st.st_mode | stat.S_IRUSR | stat.S_IWUSR) if app.config['GID'] > 0: os.chmod(path, st.st_mode | stat.S_IRGRP | stat.S_IWGRP)