q-tools/files/archive-subfolders

#!/bin/bash

VERSION="20250214"

function helpexit() {
    BS=$( basename "$0" )
    echo "Archive the subfolders of the current directory, version: $VERSION"
    echo "Requires pv, tar and python"
    echo "Usage: $BS [-z/-n/--rm] [folder name(s)]"
    echo "  -z    Compress."
    echo "  -n    No compression. [default]"
    echo "  -y    Dont ask before starting the archival."
    echo "  --nl  Do not store a separate file list."
    echo "  --rm  Remove source folders after archival."
    echo "  --split SIZE  Split archive to SIZE. ex. 500G"
    echo "  folder name: If given, archive only the named folder(s)."
    echo "               If not, for loop over all folders in the current directory."

    exit
}
function listfolders() {
   find "$@" -mindepth $DEPTH -maxdepth $DEPTH -type d | sort -V
}
function count_size() {

    cat - | python3 -c "import sys,time
def sizeof_fmt(num, suffix='B'):
    for unit in ['','K','M','G','T','P','E','Z']:
        if num < 1024.0:
            return '%3.1f%s%s' % (num, unit, suffix)
        num /= 1024.0
    return '%.1f%s%s' % (num, 'Y', suffix)
sum=0
count=0
printed=0
try:
    for line in sys.stdin:
        count += 1
        sum += int(line)
        if printed < time.time() - 1:
            sys.stderr.write('\r{}: {} #{:,}       \r'.format(sys.argv[1],sizeof_fmt(sum), count))
            sys.stderr.flush()
            printed=time.time()
    sys.stderr.write('\r{}: {} #{:,}       \r'.format(sys.argv[1],sizeof_fmt(sum), count))
except KeyboardInterrupt:
   sys.stdout.flush()
   sys.stderr.flush()
   pass

print(sum)
" "$1"
}

function exitokay() {
    [[ "$REMOVE" -eq 1 ]] && { exit; }
    echo -ne "\n${G}If all looks okay, delete source folders with: ${Z}# rm -r "
    for folder in "${REALFOLDERS[@]}"; do echo -n "'$folder' "; done
    echo ''
    exit
}

_qCol(){ true; } # incase qolop missing
. qolop &>/dev/null
Z=$( _qCol z ) #reset color
S=$( _qCol S ) #Bold
R=$( _qCol z R ) #Red
G=$( _qCol z g ) #green
Y=$( _qCol z Y ) #Yellow

NOCOMPRESSION=1
COMPRESSION=0
SUFFIX=tar
DEPTH=1
FOLDERS=( )
REMOVE=0
KEEP_LIST=1
YES=0
SPLIT=""
for ((i=1; i<=${#@}; i++)) {
    [[ "${!i}" = "-h" ]] && helpexit
    [[ "${!i}" = "--help" ]] && helpexit
    [[ "${!i}" = "-n" ]] && { continue; } # obsolete behaviour
    [[ "${!i}" = "-y" ]] && { YES=1; continue; }
    [[ "${!i}" = "-z" ]] && { COMPRESSION=1; NOCOMPRESSION=0; COMPRESSCMD="| gzip"; SUFFIX=tgz; continue; }
    [[ "${!i}" = "--nl" ]] && { KEEP_LIST=0; continue; }
    [[ "${!i}" = "--rm" ]] && { REMOVE=1; continue; }
    [[ "${!i}" = "--split" ]] && { i=$(( i + 1 )); SPLIT="${!i}"; continue; }
    [[ "${!i}" = "-"* ]] && helpexit
    FOLDERS+=( "${!i%/}" )
    DEPTH=0
}
[[ ${#FOLDERS[@]} -eq 0 ]] && FOLDERS+=( "." )
which pv &>/dev/null || { echo pv command missing; exit 1; }
if [[ -n "$SPLIT" ]]; then
  which split &>/dev/null || { echo split command missing; exit 1; }
  if [[ "$SPLIT" =~ ^[0-9]+$ ]] || [[ "$SPLIT" =~ ^[0-9]+[KMGTPEZ]$ ]] || [[ "$SPLIT" =~ ^[0-9]+[KMGTPEZ]B$ ]]; then
     true
   else
     echo "Cannot parse: $SPLIT. use ex. 500G.  valid extension K M G T P E Z"
     exit 1
  fi
fi
if tar --version | grep -q GNU; then
  SORTFILES="--sort=name"
fi

export IFS=$'\n'
REALFOLDERS=( $( listfolders "${FOLDERS[@]}" ) )
echo $R'Existing files will be overwritten!'$Z
[[ "$REMOVE" -eq 1 ]] && echo 'Source folders will be deleted!'
[[ "$KEEP_LIST" -eq 0 ]] && echo 'File list will be removed.'
[[ -n "$SPLIT" ]] && echo 'Archives split to size: '$SPLIT
echo -n $S"Archive the following folders "
[[ "$NOCOMPRESSION" -eq 1 ]] && {
    echo -n without compression;
} || {
    [[ "$COMPRESSION" -eq 1 ]] && echo -n with gzip compression
}
echo $Z
for folder in "${REALFOLDERS[@]}"; do echo "'$folder'"; done
if [[ $YES -eq 0 ]]; then
  printf "$S<ctrl-c> to quit$Z\n"
  read foo
fi
set -o pipefail

for d in "${REALFOLDERS[@]}"; do
  printf "$d"
  SIZE=$( find "$d" -type f -printf %s"\n" | count_size "$d" )
  if [[ -n "$SPLIT" ]]; then
    STORECMD="| split -d -a 4 -b $SPLIT - $d.${SUFFIX}."
  else
    STORECMD="> \"$d.${SUFFIX}\""
  fi
  printf "\n"
  eval "tar cvv $SORTFILES --index-file=\"$d.${SUFFIX}.lst\" \"$d\" | \
        pv -s \"$SIZE\" $COMPRESSCMD $STORECMD" && {
    # tar exists okay
    NEWEST="$( awk '{ print $4 " " $5 }' "$d.${SUFFIX}.lst" | sort | tail -n 1 )"
    if [[ -n "$SPLIT" ]]; then
      touch \
        -d "$NEWEST" \
        "$d.${SUFFIX}".*[0123456789] \
        "$d.${SUFFIX}.lst"
      shopt -s nullglob
      set -- "$d.${SUFFIX}".*[0123456789]
      if [[ $# -eq 1 ]]; then
         echo "Split produced only one file:"
         mv -v "$d.${SUFFIX}".*[0123456789] "$d.${SUFFIX}"
      fi
      shopt -u nullglob
    else
      touch \
        -d "$NEWEST" \
        "$d.${SUFFIX}" \
        "$d.${SUFFIX}.lst"
    fi

    if [ "$KEEP_LIST" -eq 0 ]; then
      rm -f "$d.${SUFFIX}.lst"
    fi
    if [ "$REMOVE" -eq 1 ]; then
      printf "${Y}Removing folder $d$Z\n"
      which rm-progress &> /dev/null && rm-progress -f "$d"
      [[ -d "$d" ]] && rm -r "$d"
    fi
  }
done
exitokay