#!/usr/bin/env python3 import datetime import subprocess import os import sys import shlex import json import hashlib from argparse import ArgumentParser VERSION = "0.0.9" class TB: def __init__(self, options): self.options = options self.lock = options.lock self.base_folder = options.base_folder self.backup_source = options.backup_source self.tar_args = options.tar_args self.tar_file = options.tar_file self.ssh_args = options.ssh_args self.delete_older = options.del_older self.keep_n = options.keep_n self.delete_disk_percentage = options.del_used self.config_file = os.path.join(self.base_folder, "tar-backup.txt") self.lock_file = os.path.join(self.base_folder, "tar-backup.lock") if not self.backup_source.endswith("/"): self.backup_source += "/" self.current = os.path.join(self.base_folder, "current") self.folder_format = "backup-%Y%m%d-%H%M%S" self.backup_folder = datetime.datetime.now().strftime(self.folder_format) os.makedirs(self.base_folder, exist_ok=True) self.lock_check() self.write_config() self.clean_old() success = self.make_backup() if success: self.make_softlinks() self.lock_clean() if not success: print("Program exited with errors") sys.exit(1) def diskused(self): """in percents""" pcent = subprocess.check_output( ["df", "--output=pcent", self.base_folder] ).decode("utf8") try: pcent = int("".join([x for x in pcent if x.isdigit()])) except Exception as e: return 0 return pcent def clean_old(self): print("Cleaning old backup") if self.delete_older: print("Keeping newer than {:} days".format(self.delete_older)) if self.keep_n: print("Keep at most {:} backups".format(self.keep_n)) if self.delete_disk_percentage: print("Keep until disk is {:}% full".format(self.delete_disk_percentage)) now = datetime.datetime.now() dirs = [ d for d in sorted(os.listdir(self.base_folder)) if d.startswith("backup-") and os.path.isdir(os.path.join(self.base_folder, d)) ] for i, d in enumerate(dirs): if self.delete_older: dir_date = datetime.datetime.strptime(d, self.folder_format) dir_age = now - dir_date is_old = dir_age.days > self.delete_older else: is_old = False if self.delete_disk_percentage: is_disk_full = self.diskused() > self.delete_disk_percentage else: is_disk_full = False if self.keep_n: not_kept = len(dirs) - i + 1 > self.keep_n else: not_kept = False print( "{}: {} -- old: {}, disk full: {}, keep n: {}".format( i, d, is_old, is_disk_full, not_kept ) ) if any((is_old, is_disk_full, not_kept)): print("Deleting {}".format(d)) cmd = ["rm", "-rf", os.path.join(self.base_folder, d)] subprocess.call(cmd, shell=False) def make_softlinks(self): try: os.remove(self.current) except Exception: pass if os.path.exists(os.path.join(self.base_folder, self.backup_folder)): os.symlink(self.backup_folder, self.current) def make_backup(self): if self.options.no_backup: print("Not making backup, as requested") return True os.makedirs(os.path.join(self.base_folder, self.backup_folder), exist_ok=True) print("Backing up") ssh_command = [ "ssh", *shlex.split(self.ssh_args), ] if self.ssh_args else [] with open(os.path.join(self.base_folder, self.backup_folder,self.tar_file), "wb") as fp: with open(os.path.join(self.base_folder, self.backup_folder,self.tar_file+".log"), "w") as fp_log: command = [ *ssh_command, "tar", *shlex.split(self.tar_args), self.backup_source, ] print(command) md5 = hashlib.md5() p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=fp_log) i=1 while True: chunk = p.stdout.read(1048576) if len(chunk) == 0: break print(f"{i} Mb", end='\r') i+=1 md5.update(chunk) fp.write(chunk) exitcode = p.wait() success = exitcode in (0, 2) print(f"\nWrote {os.path.join(self.base_folder, self.backup_folder,self.tar_file)}") print(f"Log file {os.path.join(self.base_folder, self.backup_folder,self.tar_file+'.log')}") with open(os.path.join(self.base_folder, self.backup_folder,self.tar_file+".md5"), "w") as fp_md5: fp_md5.write(f"{md5.hexdigest()} {self.tar_file}\n") if not success: with open(os.path.join(self.base_folder, self.backup_folder,self.tar_file+".log")) as f: print(f.read(),end='') print(f"Exit code: {exitcode}") return success def write_config(self): conf = json.dumps(vars(self.options)) with open(self.config_file, "wt") as fp: fp.write( """# Tar Backup version {version} # {conf} tar-backup \\ --del-older {older} \\ --keep-number {number} \\ --del-disk-used {used} \\ --tar-args '{args}' \\ --tar-file '{fname}' \\ --ssh-args '{ssh_args}' \\ {source} \\ {base} """.format( conf=conf, version=VERSION, older=self.delete_older, number=self.keep_n, used=self.delete_disk_percentage, args=self.tar_args, fname=self.tar_file, ssh_args=self.ssh_args, source=self.backup_source, base=self.base_folder, ) ) def lock_check(self): if self.options.lock: if os.path.exists(self.lock_file): raise FileExistsError("Lock file '{}' exists".format(self.lock_file)) with open(self.lock_file, "wt") as fp: fp.write(str(os.getpid())) def lock_clean(self): if self.options.lock: try: os.remove(self.lock_file) except FileNotFoundError: pass def get_opts(): parser = ArgumentParser( description="Backup utility using tar and ssh.", ) parser.add_argument( "--del-older", action="store", type=int, dest="del_older", default=None, help="Delete old backups older than N days", ) parser.add_argument( "--del-disk-used", action="store", type=float, dest="del_used", default=99, help="Delete old backups if disk is fuller than P percentage", ) parser.add_argument( "--keep-number", action="store", type=int, dest="keep_n", default=None, help="Keep at most N old backups", ) parser.add_argument( "--tar-args", type=str, dest="tar_args", default="-cpvv --one-file-system", help="Tar arguments. Add excludes here (example '-cp --exclude folder/'). Defaults: '%(default)s'", ) parser.add_argument( "--tar-file", type=str, dest="tar_file", default="backup.tar", help="filename for backups. Use correct extension, e.g. tgz if necessary. Defaults: '%(default)s'", ) parser.add_argument( "--ssh-args", type=str, dest="ssh_args", default=None, help="SSH arguments. If not specified, not using ssh. (example '-l user hostname'). Defaults: '%(default)s'", ) parser.add_argument( "--lock", dest="lock", action="store_true", default=False, help="Use locking to prevent concurrent backups", ) parser.add_argument( "--no-backup", dest="no_backup", action="store_true", default=False, help="Do not actually backup, only clean up.", ) parser.add_argument("--version", action="version", version=VERSION) parser.add_argument( "backup_source", action="store", help="Source folder to backup with tar+ssh" ) parser.add_argument("base_folder", action="store", help="Local backup folder") options = parser.parse_args() if "/" in options.tar_file: parser.error("--tar-file must be a filename") return options if __name__ == "__main__": opts = get_opts() TB(opts)