#!/usr/bin/env python3 import datetime import hashlib import json import os import shlex import subprocess import sys from argparse import ArgumentParser VERSION = "0.1.1" class TB: def __init__(self, options): self.options = options self.lock = options.lock self.base_folder = options.base_folder self.backup_source = options.backup_source self.tar_args = options.tar_args self.tar_file = options.tar_file self.ssh_args = options.ssh_args self.delete_older = options.del_older self.keep_n = options.keep_n self.do_gpg = options.gpg self.delete_disk_percentage = options.del_used self.config_file = os.path.join(self.base_folder, "tar-backup.txt") self.lock_file = os.path.join(self.base_folder, "tar-backup.lock") if not self.backup_source.endswith("/"): self.backup_source += "/" self.folder_format = "backup-%Y%m%d-%H%M%S" self.backup_folder = datetime.datetime.now().strftime(self.folder_format) os.makedirs(self.base_folder, exist_ok=True) self.lock_check() self.write_config() self.clean_old() success = self.make_backup() self.make_softlinks(success) self.lock_clean() if not success: print("Program exited with errors") sys.exit(1) def diskused(self): """in percents""" pcent = subprocess.check_output(["df", "--output=pcent", self.base_folder]).decode("utf8") try: pcent = int("".join([x for x in pcent if x.isdigit()])) except Exception as e: return 0 return pcent def clean_old(self): print("Cleaning old backup") if self.delete_older: print("Keeping newer than {:} days".format(self.delete_older)) if self.keep_n: print("Keep at most {:} backups".format(self.keep_n)) if self.delete_disk_percentage: print("Keep until disk is {:}% full".format(self.delete_disk_percentage)) now = datetime.datetime.now() dirs = [ d for d in sorted(os.listdir(self.base_folder)) if d.startswith("backup-") and os.path.isdir(os.path.join(self.base_folder, d)) ] for i, d in enumerate(dirs): if self.delete_older: dir_date = datetime.datetime.strptime(d, self.folder_format) dir_age = now - dir_date is_old = dir_age.days > self.delete_older else: is_old = False if self.delete_disk_percentage: is_disk_full = self.diskused() > self.delete_disk_percentage else: is_disk_full = False if self.keep_n: not_kept = len(dirs) - i + 1 > self.keep_n else: not_kept = False print("{}: {} -- old: {}, disk full: {}, keep n: {}".format(i, d, is_old, is_disk_full, not_kept)) if any((is_old, is_disk_full, not_kept)): print("Deleting {}".format(d)) cmd = ["rm", "-rf", os.path.join(self.base_folder, d)] subprocess.call(cmd, shell=False) def make_softlinks(self, success): target = "success" if success else "failed" for folder in ("latest", target): try: os.remove(os.path.join(self.base_folder, folder)) except Exception: pass if os.path.exists(os.path.join(self.base_folder, self.backup_folder)): os.symlink(self.backup_folder, os.path.join(self.base_folder, folder)) def make_backup(self): if self.options.no_backup: print("Not making backup, as requested") return True os.makedirs(os.path.join(self.base_folder, self.backup_folder), exist_ok=True) print("Backing up") ssh_command = ( [ "ssh", *shlex.split(self.ssh_args), ] if self.ssh_args else [] ) with open(os.path.join(self.base_folder, self.backup_folder, self.tar_file), "wb") as fp: with open( os.path.join(self.base_folder, self.backup_folder, self.tar_file + ".log"), "w", ) as fp_log: command = [ *ssh_command, "tar", *shlex.split(self.tar_args), self.backup_source, ] print(command) sha256 = hashlib.sha256() p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=fp_log) if self.do_gpg: pwpipe_r, pwpipe_w = os.pipe() os.write(pwpipe_w, sys.stdin.buffer.read()) os.close(pwpipe_w) gpg_command = "gpg -o - --passphrase-fd {} --batch --yes --symmetric".format(pwpipe_r) gnp = subprocess.Popen( shlex.split(gpg_command), stdin=p.stdout, stdout=subprocess.PIPE, pass_fds=[pwpipe_r] ) reader = gnp else: reader = p i = 1 while True: chunk = reader.stdout.read(1048576) if len(chunk) == 0: break print(f"{i} Mb", end="\r") i += 1 sha256.update(chunk) fp.write(chunk) if self.do_gpg: os.close(pwpipe_r) exitcode = gnp.wait() + p.wait() else: exitcode = p.wait() success = exitcode in (0,) print(f"\nWrote {os.path.join(self.base_folder, self.backup_folder,self.tar_file)}") print(f"Log file {os.path.join(self.base_folder, self.backup_folder,self.tar_file+'.log')}") with open( os.path.join(self.base_folder, self.backup_folder, self.tar_file + ".sha256"), "w", ) as fp_sha256: fp_sha256.write(f"{sha256.hexdigest()} {self.tar_file}\n") if not success: with open(os.path.join(self.base_folder, self.backup_folder, self.tar_file + ".log")) as f: print(f.read(), end="") print(f"Exit code: {exitcode}") return success def write_config(self): conf = json.dumps(vars(self.options)) with open(self.config_file, "wt") as fp: fp.write( """# Tar Backup version {version} # {conf} tar-backup \\ --del-older {older} \\ --keep-number {number} \\ --del-disk-used {used} \\ --tar-args '{args}' \\ --tar-file '{fname}' \\ --ssh-args '{ssh_args}' \\ {source} \\ {base} """.format( conf=conf, version=VERSION, older=self.delete_older, number=self.keep_n, used=self.delete_disk_percentage, args=self.tar_args, fname=self.tar_file, ssh_args=self.ssh_args, source=self.backup_source, base=self.base_folder, ) ) def lock_check(self): if self.options.lock: if os.path.exists(self.lock_file): raise FileExistsError("Lock file '{}' exists".format(self.lock_file)) with open(self.lock_file, "wt") as fp: fp.write(str(os.getpid())) def lock_clean(self): if self.options.lock: try: os.remove(self.lock_file) except FileNotFoundError: pass def get_opts(): parser = ArgumentParser( description="Backup utility using tar and ssh.", ) parser.add_argument( "--del-older", action="store", type=int, dest="del_older", default=None, help="Delete old backups older than N days", ) parser.add_argument( "--del-disk-used", action="store", type=float, dest="del_used", default=99, help="Delete old backups if disk is fuller than P percentage", ) parser.add_argument( "--keep-number", action="store", type=int, dest="keep_n", default=None, help="Keep at most N old backups", ) parser.add_argument( "--tar-args", type=str, dest="tar_args", default="-cpvv --one-file-system", help="Tar arguments. Add excludes here (example '-cp --exclude folder/'). Defaults: '%(default)s'", ) parser.add_argument( "--tar-file", type=str, dest="tar_file", default=None, help="filename for backups. Use correct extension, e.g. tgz if necessary. Defaults to [backup_write_folder].tar", ) parser.add_argument( "--ssh-args", type=str, dest="ssh_args", default=None, help="SSH arguments. If not specified, not using ssh. (example '-l user hostname'). Defaults: '%(default)s'", ) parser.add_argument( "--lock", dest="lock", action="store_true", default=False, help="Use locking to prevent concurrent backups", ) parser.add_argument( "--no-backup", dest="no_backup", action="store_true", default=False, help="Do not actually backup, only clean up.", ) parser.add_argument( "--gpg", dest="gpg", action="store_true", default=False, help="Encrypt with GPG. Read passphrase from stdin. change --tar-file to match, e.g. .gpg suffix. You might want to remove -vv from tar-args.", ) parser.add_argument("--version", action="version", version=VERSION) parser.add_argument("backup_source", action="store", help="Source folder to backup with tar+ssh") parser.add_argument("base_folder", action="store", help="Local backup folder written to") options = parser.parse_args() if options.tar_file is None: options.tar_file = os.path.basename(options.base_folder.rstrip("/")) + ".tar" if options.gpg: options.tar_file += ".gpg" if "/" in options.tar_file: parser.error("--tar-file must be a filename") return options if __name__ == "__main__": opts = get_opts() TB(opts)