307 lines
10 KiB
Python
Executable File
307 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
from argparse import ArgumentParser
|
|
|
|
VERSION = "0.1.0"
|
|
|
|
|
|
class TB:
|
|
def __init__(self, options):
|
|
self.options = options
|
|
self.lock = options.lock
|
|
self.base_folder = options.base_folder
|
|
self.backup_source = options.backup_source
|
|
self.tar_args = options.tar_args
|
|
self.tar_file = options.tar_file
|
|
self.ssh_args = options.ssh_args
|
|
self.delete_older = options.del_older
|
|
self.keep_n = options.keep_n
|
|
self.do_gpg = options.gpg
|
|
self.delete_disk_percentage = options.del_used
|
|
self.config_file = os.path.join(self.base_folder, "tar-backup.txt")
|
|
self.lock_file = os.path.join(self.base_folder, "tar-backup.lock")
|
|
|
|
if not self.backup_source.endswith("/"):
|
|
self.backup_source += "/"
|
|
self.folder_format = "backup-%Y%m%d-%H%M%S"
|
|
self.backup_folder = datetime.datetime.now().strftime(self.folder_format)
|
|
os.makedirs(self.base_folder, exist_ok=True)
|
|
|
|
self.lock_check()
|
|
self.write_config()
|
|
self.clean_old()
|
|
success = self.make_backup()
|
|
|
|
self.make_softlinks(success)
|
|
|
|
self.lock_clean()
|
|
if not success:
|
|
print("Program exited with errors")
|
|
sys.exit(1)
|
|
|
|
def diskused(self):
|
|
"""in percents"""
|
|
pcent = subprocess.check_output(["df", "--output=pcent", self.base_folder]).decode("utf8")
|
|
try:
|
|
pcent = int("".join([x for x in pcent if x.isdigit()]))
|
|
except Exception as e:
|
|
return 0
|
|
return pcent
|
|
|
|
def clean_old(self):
|
|
print("Cleaning old backup")
|
|
if self.delete_older:
|
|
print("Keeping newer than {:} days".format(self.delete_older))
|
|
if self.keep_n:
|
|
print("Keep at most {:} backups".format(self.keep_n))
|
|
if self.delete_disk_percentage:
|
|
print("Keep until disk is {:}% full".format(self.delete_disk_percentage))
|
|
|
|
now = datetime.datetime.now()
|
|
dirs = [
|
|
d
|
|
for d in sorted(os.listdir(self.base_folder))
|
|
if d.startswith("backup-") and os.path.isdir(os.path.join(self.base_folder, d))
|
|
]
|
|
for i, d in enumerate(dirs):
|
|
if self.delete_older:
|
|
dir_date = datetime.datetime.strptime(d, self.folder_format)
|
|
dir_age = now - dir_date
|
|
is_old = dir_age.days > self.delete_older
|
|
else:
|
|
is_old = False
|
|
if self.delete_disk_percentage:
|
|
is_disk_full = self.diskused() > self.delete_disk_percentage
|
|
else:
|
|
is_disk_full = False
|
|
if self.keep_n:
|
|
not_kept = len(dirs) - i + 1 > self.keep_n
|
|
else:
|
|
not_kept = False
|
|
|
|
print("{}: {} -- old: {}, disk full: {}, keep n: {}".format(i, d, is_old, is_disk_full, not_kept))
|
|
if any((is_old, is_disk_full, not_kept)):
|
|
print("Deleting {}".format(d))
|
|
cmd = ["rm", "-rf", os.path.join(self.base_folder, d)]
|
|
subprocess.call(cmd, shell=False)
|
|
|
|
def make_softlinks(self, success):
|
|
target = "success" if success else "failed"
|
|
for folder in ("latest", target):
|
|
try:
|
|
os.remove(os.path.join(self.base_folder, folder))
|
|
except Exception:
|
|
pass
|
|
if os.path.exists(os.path.join(self.base_folder, self.backup_folder)):
|
|
os.symlink(self.backup_folder, os.path.join(self.base_folder, folder))
|
|
|
|
def make_backup(self):
|
|
if self.options.no_backup:
|
|
print("Not making backup, as requested")
|
|
return True
|
|
os.makedirs(os.path.join(self.base_folder, self.backup_folder), exist_ok=True)
|
|
print("Backing up")
|
|
|
|
ssh_command = (
|
|
[
|
|
"ssh",
|
|
*shlex.split(self.ssh_args),
|
|
]
|
|
if self.ssh_args
|
|
else []
|
|
)
|
|
|
|
with open(os.path.join(self.base_folder, self.backup_folder, self.tar_file), "wb") as fp:
|
|
with open(
|
|
os.path.join(self.base_folder, self.backup_folder, self.tar_file + ".log"),
|
|
"w",
|
|
) as fp_log:
|
|
command = [
|
|
*ssh_command,
|
|
"tar",
|
|
*shlex.split(self.tar_args),
|
|
self.backup_source,
|
|
]
|
|
print(command)
|
|
md5 = hashlib.md5()
|
|
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=fp_log)
|
|
|
|
if self.do_gpg:
|
|
pwpipe_r, pwpipe_w = os.pipe()
|
|
os.write(pwpipe_w, sys.stdin.buffer.read())
|
|
os.close(pwpipe_w)
|
|
gpg_command = "gpg -o - --passphrase-fd {} --batch --yes --symmetric".format(pwpipe_r)
|
|
gnp = subprocess.Popen(
|
|
shlex.split(gpg_command), stdin=p.stdout, stdout=subprocess.PIPE, pass_fds=[pwpipe_r]
|
|
)
|
|
reader = gnp
|
|
else:
|
|
reader = p
|
|
|
|
i = 1
|
|
while True:
|
|
chunk = reader.stdout.read(1048576)
|
|
if len(chunk) == 0:
|
|
break
|
|
print(f"{i} Mb", end="\r")
|
|
i += 1
|
|
md5.update(chunk)
|
|
fp.write(chunk)
|
|
if self.do_gpg:
|
|
os.close(pwpipe_r)
|
|
exitcode = gnp.wait() + p.wait()
|
|
else:
|
|
exitcode = p.wait()
|
|
success = exitcode in (0,)
|
|
print(f"\nWrote {os.path.join(self.base_folder, self.backup_folder,self.tar_file)}")
|
|
print(f"Log file {os.path.join(self.base_folder, self.backup_folder,self.tar_file+'.log')}")
|
|
with open(
|
|
os.path.join(self.base_folder, self.backup_folder, self.tar_file + ".md5"),
|
|
"w",
|
|
) as fp_md5:
|
|
fp_md5.write(f"{md5.hexdigest()} {self.tar_file}\n")
|
|
if not success:
|
|
with open(os.path.join(self.base_folder, self.backup_folder, self.tar_file + ".log")) as f:
|
|
print(f.read(), end="")
|
|
print(f"Exit code: {exitcode}")
|
|
return success
|
|
|
|
def write_config(self):
|
|
conf = json.dumps(vars(self.options))
|
|
with open(self.config_file, "wt") as fp:
|
|
fp.write(
|
|
"""# Tar Backup version {version}
|
|
# {conf}
|
|
tar-backup \\
|
|
--del-older {older} \\
|
|
--keep-number {number} \\
|
|
--del-disk-used {used} \\
|
|
--tar-args '{args}' \\
|
|
--tar-file '{fname}' \\
|
|
--ssh-args '{ssh_args}' \\
|
|
{source} \\
|
|
{base}
|
|
""".format(
|
|
conf=conf,
|
|
version=VERSION,
|
|
older=self.delete_older,
|
|
number=self.keep_n,
|
|
used=self.delete_disk_percentage,
|
|
args=self.tar_args,
|
|
fname=self.tar_file,
|
|
ssh_args=self.ssh_args,
|
|
source=self.backup_source,
|
|
base=self.base_folder,
|
|
)
|
|
)
|
|
|
|
def lock_check(self):
|
|
if self.options.lock:
|
|
if os.path.exists(self.lock_file):
|
|
raise FileExistsError("Lock file '{}' exists".format(self.lock_file))
|
|
with open(self.lock_file, "wt") as fp:
|
|
fp.write(str(os.getpid()))
|
|
|
|
def lock_clean(self):
|
|
if self.options.lock:
|
|
try:
|
|
os.remove(self.lock_file)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
def get_opts():
|
|
parser = ArgumentParser(
|
|
description="Backup utility using tar and ssh.",
|
|
)
|
|
parser.add_argument(
|
|
"--del-older",
|
|
action="store",
|
|
type=int,
|
|
dest="del_older",
|
|
default=None,
|
|
help="Delete old backups older than N days",
|
|
)
|
|
parser.add_argument(
|
|
"--del-disk-used",
|
|
action="store",
|
|
type=float,
|
|
dest="del_used",
|
|
default=99,
|
|
help="Delete old backups if disk is fuller than P percentage",
|
|
)
|
|
parser.add_argument(
|
|
"--keep-number",
|
|
action="store",
|
|
type=int,
|
|
dest="keep_n",
|
|
default=None,
|
|
help="Keep at most N old backups",
|
|
)
|
|
parser.add_argument(
|
|
"--tar-args",
|
|
type=str,
|
|
dest="tar_args",
|
|
default="-cpvv --one-file-system",
|
|
help="Tar arguments. Add excludes here (example '-cp --exclude folder/'). Defaults: '%(default)s'",
|
|
)
|
|
parser.add_argument(
|
|
"--tar-file",
|
|
type=str,
|
|
dest="tar_file",
|
|
default=None,
|
|
help="filename for backups. Use correct extension, e.g. tgz if necessary. Defaults to [backup_write_folder].tar",
|
|
)
|
|
parser.add_argument(
|
|
"--ssh-args",
|
|
type=str,
|
|
dest="ssh_args",
|
|
default=None,
|
|
help="SSH arguments. If not specified, not using ssh. (example '-l user hostname'). Defaults: '%(default)s'",
|
|
)
|
|
parser.add_argument(
|
|
"--lock",
|
|
dest="lock",
|
|
action="store_true",
|
|
default=False,
|
|
help="Use locking to prevent concurrent backups",
|
|
)
|
|
parser.add_argument(
|
|
"--no-backup",
|
|
dest="no_backup",
|
|
action="store_true",
|
|
default=False,
|
|
help="Do not actually backup, only clean up.",
|
|
)
|
|
parser.add_argument(
|
|
"--gpg",
|
|
dest="gpg",
|
|
action="store_true",
|
|
default=False,
|
|
help="Encrypt with GPG. Read passphrase from stdin. change --tar-file to match, e.g. .gpg suffix. You might want to remove -vv from tar-args.",
|
|
)
|
|
parser.add_argument("--version", action="version", version=VERSION)
|
|
|
|
parser.add_argument("backup_source", action="store", help="Source folder to backup with tar+ssh")
|
|
parser.add_argument("base_folder", action="store", help="Local backup folder written to")
|
|
options = parser.parse_args()
|
|
if options.tar_file is None:
|
|
options.tar_file = os.path.basename(options.base_folder.rstrip("/")) + ".tar"
|
|
if options.gpg:
|
|
options.tar_file += ".gpg"
|
|
if "/" in options.tar_file:
|
|
parser.error("--tar-file must be a filename")
|
|
return options
|
|
|
|
|
|
if __name__ == "__main__":
|
|
opts = get_opts()
|
|
TB(opts)
|