moonq/flees
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

too much stuff. uploader script from template. dropdown menu for tools

Browse Source
This commit is contained in:
Ville Rantanen
2018-03-15 12:14:32 +02:00
parent 6724ac2557
commit 86c855285c
6 changed files with 340 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
code/app.py
View File

@@ -4,7 +4,8 @@ import os,sys,time,stat
import json
from datetime import datetime
from flask import Flask, render_template, jsonify, current_app, Response, \
redirect, url_for, request, g, session, send_file, send_from_directory
redirect, url_for, request, g, session, send_file, send_from_directory, \
abort
from werkzeug.utils import secure_filename
import zipfile
from multiprocessing import Process
@@ -98,7 +99,7 @@ def upload(name = None, token = None):
if file:
filename = os.path.join(
share['path'],
secure_filename(
secure_filename_hidden(
file.filename
)
)
@@ -133,10 +134,10 @@ def upload_join_splitted(name, token):
if not 'filename' in request.form:
return "No filename given", 400
parts = []
for part in range(100):
for part in range(500):
filename = os.path.join(
share['path'],
"%s.part.%03d"%(
".%s.part.%03d"%(
request.form['filename'],
part
)
@@ -145,7 +146,7 @@ def upload_join_splitted(name, token):
parts.append(filename)
part_existed = part
if len(parts) == 0:
return "Invalid partial filename\n", 400
return "Invalid partial filename %s -> %s\n"%( request.form['filename'], filename), 400
if not len(parts) == part_existed + 1:
return "Parts missing\n", 400
target_name = os.path.join(
@@ -163,6 +164,41 @@ def upload_join_splitted(name, token):
return "Joining started\n", 200
@app.route('/upload/url', methods=['POST'])
def upload_url():
if request.method == 'POST':
name = request.form['name']
url = request.form['url']
if not is_valid_url(url):
return "URL not valid", 400
(ok,share) = get_share(name)
if not ok:
return share
if not get_or_none('upload', share) == True:
return "Upload not allowed\n",400
filename = os.path.join(
share['path'],
secure_filename(
os.path.basename(url)
)
)
if os.path.exists(filename):
file_versionize(filename)
download_url(url, filename)
set_rights(filename)
notify({
"recipient": get_or_none('recipient', share),
"share": name,
"filename": filename,
"operation": "upload"
})
if 'from_gui' in request.form:
if request.form['from_gui'] == "true":
return redirect(url_for('list_view',name=name))
return "File uploaded\n", 200
@app.route('/send/<name>', methods=['GET'])
def send(name):
(ok,share) = get_share(name)
@@ -171,7 +207,22 @@ def send(name):
return render_template('send.html',name=name)
@app.route('/files/<name>/<token>', methods=['GET'])
@app.route('/file/size/<name>/<token>/<path:filename>', methods=['GET'])
def file_size(name, token, filename):
(ok,share) = get_share(name, token = token)
if not ok:
return share
full_path = os.path.join(
share['path'],
secure_filename_hidden(filename)
)
if not os.path.exists(full_path):
return "-1", 200
size = os.stat(full_path).st_size
return str(size), 200
@app.route('/file/list/<name>/<token>', methods=['GET'])
def list_files(name, token):
(ok,share) = get_share(name, token = token)
if not ok:
@@ -443,60 +494,99 @@ def script_upload_split(name = None, token = None):
return share
if not get_or_none('upload', share) == True:
return "Upload not allowed",400
return """#!/bin/bash
test -n "$1" || {
echo "Usage: [-s SplitMegabytes] file/folder [files/folders]"
exit 1
}
MAXBYTES=$(( 512 * 1024 * 1024 ))
for (( i=1; i<=$#; i++ )); do
j=$(( $i + 1 ))
[[ ${!i} = "-s" ]] && {
MAXBYTES=$(( ${!j} * 1024 * 1024 ))
shift 2
}
done
CAT=$( which cat )
which pv &> /dev/null && CAT=$( which pv )
ROOTURL="%s"
SHARE="%s"
TOKEN="%s"
send_file() {
$CAT "$file_name" | split -d -a 3 -b $MAXBYTES \
--filter="curl -f -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \
- "$base_name.part." && \
curl -F "filename=$base_name" ${ROOTURL}upload_join/${SHARE}/${TOKEN}
}
send_folder() {
which pv &> /dev/null && printf -v dusize -- "--size %%dk" $( du -s -k "$file_name" | cut -f1 )
tar c "$file_name" | $CAT $dusize - | split -d -a 3 -b $MAXBYTES \
--filter="curl -f -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \
- "$base_name.tar.part." && \
curl -F "filename=$base_name.tar" ${ROOTURL}upload_join/${SHARE}/${TOKEN}
}
echo "Splitting to $(( $MAXBYTES / 1024 / 1024 )) Mb chunks"
for file_name in "$@"; do
base_name=$( basename "$file_name" )
test -f "$file_name" && {
printf "Sending file: %%s\n" "$file_name"
send_file
continue
}
test -d "$file_name" && {
printf "Sending folder: %%s\n" "$file_name"
send_folder
continue
}
done
"""%(
request.url_root,
name,
token
return render_template(
"upload_split.py",
name = name,
token = token,
rooturl = request.url_root
)
#~ mmmmm # ""#
#~ # "# mmm mmm m m mmmmm mmm #mmm # mmm
#~ #mmmm" #" # # " # # # # # " # #" "# # #" #
#~ # "m #"""" """m # # # # # m"""# # # # #""""
#~ # " "#mm" "mmm" "mm"# # # # "mm"# ##m#" "mm "#mm"
#~ @app.route("/resumable/send")
#~ def resumable_example():
#~ return render_template("resumable_upload.html")
#~ # resumable.js uses a GET request to check if it uploaded the file already.
#~ # NOTE: your validation here needs to match whatever you do in the POST (otherwise it will NEVER find the files)
#~ @app.route("/resumable/get", methods=['GET'])
#~ def resumable():
#~ resumableIdentfier = request.args.get('resumableIdentifier', type=str)
#~ resumableFilename = request.args.get('resumableFilename', type=str)
#~ resumableChunkNumber = request.args.get('resumableChunkNumber', type=int)
#~ if not resumableIdentfier or not resumableFilename or not resumableChunkNumber:
#~ # Parameters are missing or invalid
#~ abort(500, 'Parameter error')
#~ # chunk folder path based on the parameters
#~ temp_dir = os.path.join(temp_base, resumableIdentfier)
#~ # chunk path based on the parameters
#~ chunk_file = os.path.join(temp_dir, get_chunk_name(resumableFilename, resumableChunkNumber))
#~ app.logger.debug('Getting chunk: %s', chunk_file)
#~ if os.path.isfile(chunk_file):
#~ # Let resumable.js know this chunk already exists
#~ return 'OK'
#~ else:
#~ # Let resumable.js know this chunk does not exists and needs to be uploaded
#~ abort(404, 'Not found')
#~ # if it didn't already upload, resumable.js sends the file here
#~ @app.route("/resumable/post", methods=['POST'])
#~ def resumable_post():
#~ resumableTotalChunks = request.form.get('resumableTotalChunks', type=int)
#~ resumableChunkNumber = request.form.get('resumableChunkNumber', default=1, type=int)
#~ resumableFilename = request.form.get('resumableFilename', default='error', type=str)
#~ resumableIdentfier = request.form.get('resumableIdentifier', default='error', type=str)
#~ # get the chunk data
#~ chunk_data = request.files['file']
#~ # make our temp directory
#~ temp_dir = os.path.join(temp_base, resumableIdentfier)
#~ if not os.path.isdir(temp_dir):
#~ os.makedirs(temp_dir, 0777)
#~ # save the chunk data
#~ chunk_name = get_chunk_name(resumableFilename, resumableChunkNumber)
#~ chunk_file = os.path.join(temp_dir, chunk_name)
#~ chunk_data.save(chunk_file)
#~ app.logger.debug('Saved chunk: %s', chunk_file)
#~ # check if the upload is complete
#~ chunk_paths = [os.path.join(temp_dir, get_chunk_name(resumableFilename, x)) for x in range(1, resumableTotalChunks+1)]
#~ upload_complete = all([os.path.exists(p) for p in chunk_paths])
#~ # combine all the chunks to create the final file
#~ if upload_complete:
#~ target_file_name = os.path.join(temp_base, resumableFilename)
#~ with open(target_file_name, "ab") as target_file:
#~ for p in chunk_paths:
#~ stored_chunk_file_name = p
#~ stored_chunk_file = open(stored_chunk_file_name, 'rb')
#~ target_file.write(stored_chunk_file.read())
#~ stored_chunk_file.close()
#~ os.unlink(stored_chunk_file_name)
#~ target_file.close()
#~ os.rmdir(temp_dir)
#~ app.logger.debug('File saved to: %s', target_file_name)
#~ return 'OK'
#~ def get_chunk_name(uploaded_filename, chunk_number):
#~ return uploaded_filename + "_part_%03d" % chunk_number
class uploadJoiner:
def __init__(self, target_name, parts):
self.target_name = target_name
@@ -623,6 +713,13 @@ def notify(msg):
app.config['notifier'].notify(msg)
def secure_filename_hidden(filename):
secure = secure_filename(filename)
if filename.startswith("."):
secure = "." + secure
return secure
def set_rights(path):
os.chown(path, app.config['UID'], app.config['GID'])
st = os.stat(path)

19
code/static/css/styles.css
View File

@@ -102,8 +102,23 @@ tr:nth-child(odd) {
width: 200px;
}
#list_upload_button {
position: absolute;
right: 8px;
float: right;
}
#list_url_upload_text {
width: 95%;
float: right;
}
#list_url_upload_button {
float: right;
}
#list_info_toggle {
margin-top: 24px;
cursor: pointer;
text-align: right;
}
#list_info {
display: none;
}
#list_table {

15
code/static/js/scripts.js
View File

@@ -1,4 +1,12 @@
function clear_text(id,defaultValue) {
var element = document.getElementById(id);
if ( element.value == defaultValue ) {
element.value = '';
}
}
function index_form_enter(event) {
if (event.which || event.keyCode) {
if ((event.which == 13) || (event.keyCode == 13)) {
@@ -12,6 +20,13 @@ function index_form_submit() {
document.getElementById("index_form_name").value;
}
function infoToggle() {
var info = document.getElementById("list_info");
info.style.display = info.style.display === 'block' ? 'none' : 'block';
}
function UploadFile(file,file_no,files_total) {
if (uploadTurn != file_no) {
// Wait for our turn to upload. check every 0.5s

43
code/templates/list.html
View File

@@ -2,45 +2,56 @@
{% block body %}
<div id=list_menu>
{% if upload %}
<div id=list_upload>
<div id=list_upload>
<form id="upload_form" action={{ url_for('upload') }} method=post enctype=multipart/form-data>
<input id="list_upload_name" type=hidden name=name value="{{ name|safe }}" />
<input type=hidden name=from_gui value="true" />
<input id="list_upload_select" type=file name=file
onchange="FileSelectHandler(event)" multiple="multiple"><br>
<input id="list_upload_button" type=submit value=Upload>
<div id="progress"></div>
<div id="progress" class="clear"></div>
</form>
</div>
</div>
{% else %}
<div id=list_upload>
<input type=file name=file disabled>
<input type=submit value=Upload disabled>
</div>
</div>
{% endif %}
<div id=list_info>
Share:
<ul>
<div id=list_info_toggle onclick="infoToggle()">Share tools</div>
<div id=list_info>
<div id=list_url_upload>
<form id="url_upload_form" action={{ url_for('upload_url') }} method=post>
<input id="list_upload_name" type=hidden name=name value="{{ name|safe }}" />
<input type=hidden name=from_gui value="true" />
<input id="list_url_upload_text" type=text name=url
value="https://..." onclick="clear_text(this.id,'https://...')"><br>
<input id="list_url_upload_button" type=submit value="Upload URL">
<div class="clear"></div>
</form>
</div>
<ul>
{% if public %}
<li>is <a href="{{ url_for('index') }}" title="Share is publicly visible in the index">public</a>
<li>is <a href="{{ url_for('index') }}" title="Share is publicly visible in the index">public</a>
{% else %}
<li>is <a href="{{ url_for('index') }}" title="Share not listed in the index unless you have logged in">unlisted</a>
<li>is <a href="{{ url_for('index') }}" title="Share not listed in the index unless you have logged in">unlisted</a>
{% endif %}
{% if expire %}
<li><span title="All share operations disabled after the date">expires {{ expire }}</span>
<li><span title="All share operations disabled after the date">expires {{ expire }}</span>
{% else %}
<li>never expires
<li>never expires
{% endif %}
{% if upload %}
{% if overwrite %}
<li><span title="Uploaded files with existing names are overwritten">uploads overwrite</span>
<li><span title="Uploaded files with existing names are overwritten">uploads overwrite</span>
{% else %}
<li><span title="Uploaded files with existing names are versioned">uploads versioned</span>
<li><span title="Uploaded files with existing names are versioned">uploads versioned</span>
{% endif %}
{% endif %}
<li><a href="{{ url_for('download_zip',name=name) }}" title="Download all the files as one ZIP file. Total size of files must be less than {{ g.max_zip_size }} Mb">Download as zip</a>
<li><a href="{{ url_for('logout',name=name) }}">Logout</a>
</div>
<li><a href="{{ url_for('download_zip',name=name) }}" title="Download all the files as one ZIP file. Total size of files must be less than {{ g.max_zip_size }} Mb">Download as zip</a>
<li><a href="{{ url_for('logout',name=name) }}">Logout</a>
</div>
</div>
<div id=list_left>
<div id=list_title><h1>{{ name }}</h1></div>

109
code/templates/upload_split.py Normal file
View File

@@ -0,0 +1,109 @@
#!/usr/bin/env python
import argparse,sys,os, subprocess
from subprocess import Popen, PIPE, STDOUT
from io import BytesIO
ROOTURL="{{ rooturl }}"
SHARE="{{ name }}"
TOKEN="{{ token }}"
def split_upload(path, opts):
if os.path.isdir(path):
tar = Popen(
[
'tar','c',path
],
stdout = PIPE
)
reader = tar.stdout
basename = os.path.basename(path.rstrip("/")) + ".tar"
elif os.path.isfile(path):
reader = open(path, 'rb')
basename = os.path.basename(path)
else:
print("Path %s doesnt exist"%( path, ))
return
try:
chunk = reader.read(opts.split * 1024 * 1024)
part = 0
while chunk != "":
chunk_name = ".%s.part.%03d"%(
basename,
part
)
print("%s part %d"%( basename, part ))
if not is_chunk_sent(chunk_name, opts):
p = Popen(
[
'curl','-f',
'-F','file=@-;filename=%s'%(chunk_name,),
'%supload/%s/%s'%(opts.rooturl, opts.share, opts.token)
],
stdout=PIPE,
stdin=PIPE,
stderr=PIPE
)
stdout_data, stderr_data = p.communicate(input=chunk)
print(stdout_data)
chunk = reader.read(opts.split * 1024 * 1024)
part += 1
finally:
reader.close()
join_chunks(basename,opts)
return
def is_chunk_sent(name, opts):
p = Popen(
[
'curl','-s',
'%sfile/size/%s/%s/%s'%(opts.rooturl, opts.share, opts.token, name)
],
stdout=PIPE,
stderr=PIPE
)
stdout_data, stderr_data = p.communicate()
return stdout_data == str(opts.split * 1024 * 1024)
def join_chunks(name,opts):
p = Popen(
[
'curl',
'-F','filename=%s'%(name,),
'%supload_join/%s/%s'%(opts.rooturl, opts.share, opts.token)
],
stdout=PIPE,
stderr=PIPE
)
stdout_data, stderr_data = p.communicate()
print(stdout_data)
def parse_options():
parser = argparse.ArgumentParser(description='Flees uploader')
parser.add_argument('-s', action="store", type=int, dest="split", default = 64,
help = "Split size in megabytes [%(default)s]"
)
parser.add_argument('--rooturl', action="store", dest="rooturl",
default = ROOTURL,
help = "Address of Flees server [%(default)s]"
)
parser.add_argument('--share', action="store", dest="share",
default = SHARE,
help = "Name of Flees share [%(default)s]"
)
parser.add_argument('--token', action="store", dest="token",
default = TOKEN,
help = "API token for the share [%(default)s]"
)
parser.add_argument('path', nargs='+')
return parser.parse_args()
if __name__ == "__main__":
opts = parse_options()
for path in opts.path:
split_upload(path,opts)

18
code/utils/utils.py
View File

@@ -3,8 +3,18 @@ from datetime import datetime
from flask import current_app as app
try:
from urllib.request import pathname2url
from urllib.request import URLopener
from urllib.request import urlparse
except ImportError:
from urllib import pathname2url
from urllib import URLopener
from urlparse import urlparse
def download_url(url,filename):
downloader = URLopener()
downloader.retrieve(url, filename)
return
def file_date_human(num):
return datetime.fromtimestamp(
@@ -61,6 +71,14 @@ def is_path_safe(path):
return True
def is_valid_url(url, qualifying = None):
min_attributes = ('scheme', 'netloc')
qualifying = min_attributes if qualifying is None else qualifying
token = urlparse(url)
return all([getattr(token, qualifying_attr)
for qualifying_attr in qualifying])
def iter_folder_files(path, recursive = True):
if recursive:
for dirpath, dirnames, filenames in os.walk(path, topdown = False):