added splitted file uploader and joiner. its still a bit beta

This commit is contained in:
Ville Rantanen
2018-02-24 00:13:18 +02:00
parent 7d424549a2
commit c0dbe045ec
2 changed files with 148 additions and 13 deletions

View File

@@ -4,14 +4,15 @@
import os,sys,time,stat
import json
from datetime import datetime
from flask import Flask, render_template, jsonify, current_app, \
from flask import Flask, render_template, jsonify, current_app, Response, \
redirect, url_for, request, g, session, send_file, send_from_directory
from werkzeug.utils import secure_filename
import hashlib
import zipfile
from multiprocessing import Process
from revprox import ReverseProxied
__FLEES_VERSION__ = "20180221.0"
__FLEES_VERSION__ = "20180224.0b"
app = Flask(__name__)
app.config.from_object(__name__)
# Read config from json !
@@ -90,7 +91,7 @@ def upload(name = None, password = None):
if not ok:
return share
if not get_or_none(share,'upload') == True:
return "Upload not allowed",400
return "Upload not allowed\n",400
if file:
filename = os.path.join(
share['path'],
@@ -115,7 +116,53 @@ def upload(name = None, password = None):
return redirect(url_for('list_view',name=name))
return "File uploaded\n", 200
else:
return "Use the 'file' variable to upload",400
return "Use the 'file' variable to upload\n",400
@app.route('/upload_join/<name>/<password>', methods=['POST'])
def upload_join_splitted(name, password):
if request.method == 'POST':
session[name] = password
(ok,share) = get_share(name)
if not ok:
return share
if not get_or_none(share,'upload') == True:
return "Upload not allowed",400
if not 'filename' in request.form:
return "No filename given", 400
parts = []
for part in range(100):
filename = os.path.join(
share['path'],
"%s.part.%02d"%(
request.form['filename'],
part
)
)
if os.path.exists(filename):
parts.append(filename)
part_existed = part
if len(parts) == 0:
return "Invalid partial filename\n", 400
if not len(parts) == part_existed + 1:
return "Parts missing\n", 400
target_name = os.path.join(
share['path'],
request.form['filename']
)
if get_or_none(share, 'overwrite') == False:
if os.path.exists(target_name):
file_versionize(target_name)
try:
begin = uploadJoiner(target_name, parts)
except:
return "Joining failed\n", 400
return "Joining started\n", 200
#~ return Response(joiner(target_name, parts), mimetype="text/plain", content_type="text/event-stream")
#~ return "%d parts joined"%(len(parts),), 200
@app.route('/send/<name>', methods=['GET'])
def send(name):
@@ -399,6 +446,84 @@ get_file() {
return script
@app.route('/script/upload_split/<name>/<password>', methods=['GET'])
def script_upload_split(name = None, password = None):
session[name] = password
(ok,share) = get_share(name)
if not ok:
return share
if not get_or_none(share,'upload') == True:
return "Upload not allowed",400
return """#!/bin/bash
test -n "$1" || {
echo "First argument is split size in megabytes, add files to upload as next arguments"
exit 1
}
test -n "$2" || {
echo "Add files to upload as argument"
exit 1
}
CAT=$( which cat )
which pv &> /dev/null && CAT=$( which pv )
ROOTURL="%s"
SHARE="%s"
TOKEN="%s"
MAXBYTES=$(( $1 * 1024 * 1024 ))
shift 1
send_file() {
$CAT "$file_name" | split -d -b $MAXBYTES \
--filter="curl -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \
- "$base_name.part."
curl -F "filename=$base_name" ${ROOTURL}upload_join/${SHARE}/${TOKEN}
}
send_folder() {
which pv &> /dev/null && printf -v dusize -- "--size %%dk" $( du -s -k "$file_name" | cut -f1 )
tar c "$file_name" | $CAT $dusize - | split -d -b $MAXBYTES \
--filter="curl -F \\"file=@-;filename=\${FILE}\\" ${ROOTURL}upload/${SHARE}/${TOKEN}" \
- "$base_name.tgz.part."
curl -F "filename=$base_name.tgz" ${ROOTURL}upload_join/${SHARE}/${TOKEN}
}
for file_name in "$@"; do
base_name=$( basename "$file_name" )
test -f "$file_name" && {
printf "Sending file: %%s\n" "$file_name"
send_file
continue
}
test -d "$file_name" && {
printf "Sending folder: %%s\n" "$file_name"
send_folder
continue
}
done
"""%(
request.url_root,
name,
password
)
class uploadJoiner:
def __init__(self, target_name, parts):
self.target_name = target_name
self.parts = parts
self.chunk_size = 10 * 1024 * 1024
p = Process(target=self.run, args=())
p.daemon = True
p.start()
def run(self):
with open(self.target_name,'wb') as writer:
for part in self.parts:
with open(part,'rb') as reader:
for chunk in iter(lambda: reader.read(self.chunk_size), b""):
writer.write(chunk)
set_rights(self.target_name)
for part in self.parts:
os.remove(part)
def file_stat(filename):
s = os.stat(filename)
return {
@@ -426,17 +551,23 @@ def file_date_human(num):
).strftime(app.config['DATE_FORMAT'])
def file_versionize(filename):
""" Move file to old version """
""" Move file to versioned with integer """
stats = file_stat(filename)
basename, extension = os.path.splitext(stats['name'])
new_name = os.path.join(
os.path.dirname(filename),
secure_filename("%s.%s%s"%(
basename,
stats['mtime'],
extension
))
)
version = 1
while True:
new_name = os.path.join(
os.path.dirname(filename),
secure_filename("%s.v%d%s"%(
basename,
version,
extension
))
)
if os.path.exists(new_name):
version += 1
else:
break
os.rename(filename,new_name)

View File

@@ -151,3 +151,7 @@ tr:nth-child(odd) {
#progress.failure {
color: red;
}
.sortarrow {
font-size: large;
}