Merge branch 'master' of https://git.six9.net/moonq/q-tools

files/archive-subfolders

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-VERSION="20230307"
+VERSION="20250208"
 
 function helpexit() {
     BS=$( basename "$0" )
@@ -12,6 +12,7 @@ function helpexit() {
     echo "  -y    Dont ask before starting the archival."
     echo "  --nl  Do not store a separate file list."
     echo "  --rm  Remove source folders after archival."
+    echo "  --split SIZE  Split archive to SIZE. ex. 500G"
     echo "  folder name: If given, archive only the named folder(s)."
     echo "               If not, for loop over all folders in the current directory."
 
@@ -22,7 +23,7 @@ function listfolders() {
 }
 function count_size() {
 
-    cat - | python3 -c "import sys
+    cat - | python3 -c "import sys,time
 def sizeof_fmt(num, suffix='B'):
     for unit in ['','K','M','G','T','P','E','Z']:
         if num < 1024.0:
@@ -30,15 +31,22 @@ def sizeof_fmt(num, suffix='B'):
         num /= 1024.0
     return '%.1f%s%s' % (num, 'Y', suffix)
 sum=0
+count=0
+printed=0
 try:
     for line in sys.stdin:
+        count += 1
         sum += int(line)
-        sys.stderr.write('\r%s: %s        \r'%(sys.argv[1],sizeof_fmt(sum)))
-        sys.stderr.flush()
+        if printed < time.time() - 1:
+            sys.stderr.write('\r{}: {} #{:,}       \r'.format(sys.argv[1],sizeof_fmt(sum), count))
+            sys.stderr.flush()
+            printed=time.time()
+    sys.stderr.write('\r{}: {} #{:,}       \r'.format(sys.argv[1],sizeof_fmt(sum), count))
 except KeyboardInterrupt:
    sys.stdout.flush()
    sys.stderr.flush()
    pass
+
 print(sum)
 " "$1"
 }
@@ -67,6 +75,7 @@ FOLDERS=( )
 REMOVE=0
 KEEP_LIST=1
 YES=0
+SPLIT=""
 for ((i=1; i<=${#@}; i++)) {
     [[ "${!i}" = "-h" ]] && helpexit
     [[ "${!i}" = "--help" ]] && helpexit
@@ -75,12 +84,16 @@ for ((i=1; i<=${#@}; i++)) {
     [[ "${!i}" = "-z" ]] && { COMPRESSION=1; NOCOMPRESSION=0; COMPRESSCMD="| gzip"; SUFFIX=tgz; continue; }
     [[ "${!i}" = "--nl" ]] && { KEEP_LIST=0; continue; }
     [[ "${!i}" = "--rm" ]] && { REMOVE=1; continue; }
+    [[ "${!i}" = "--split" ]] && { i=$(( i + 1 )); SPLIT="${!i}"; continue; }
     [[ "${!i}" = "-"* ]] && helpexit
     FOLDERS+=( "${!i%/}" )
     DEPTH=0
 }
 [[ ${#FOLDERS[@]} -eq 0 ]] && FOLDERS+=( "." )
 which pv &>/dev/null || { echo pv command missing; exit 1; }
+if [[ -n "$SPLIT" ]]; then
+  which split &>/dev/null || { echo split command missing; exit 1; }
+fi
 if tar --version | grep -q GNU; then
   SORTFILES="--sort=name"
 fi
@@ -90,6 +103,7 @@ REALFOLDERS=( $( listfolders "${FOLDERS[@]}" ) )
 echo $R'Existing files will be overwritten!'$Z
 [[ "$REMOVE" -eq 1 ]] && echo 'Source folders will be deleted!'
 [[ "$KEEP_LIST" -eq 0 ]] && echo 'File list will be removed.'
+[[ -n "$SPLIT" ]] && echo 'Archives split to size: '$SPLIT
 echo -n $S"Archive the following folders "
 [[ "$NOCOMPRESSION" -eq 1 ]] && {
     echo -n without compression;
@@ -107,15 +121,27 @@ set -o pipefail
 for d in "${REALFOLDERS[@]}"; do
   printf "$d"
   SIZE=$( find "$d" -type f -printf %s"\n" | count_size "$d" )
+  if [[ -n "$SPLIT" ]]; then
+    STORECMD="| split -d -a 4 -b $SPLIT - $d.${SUFFIX}."
+  else
+    STORECMD="> \"$d.${SUFFIX}\""
+  fi
   printf "\n"
   eval "tar cvv $SORTFILES --index-file=\"$d.${SUFFIX}.lst\" \"$d\" | \
-        pv -s \"$SIZE\" $COMPRESSCMD > \"$d.${SUFFIX}\"" && {
+        pv -s \"$SIZE\" $COMPRESSCMD $STORECMD" && {
     # tar exists okay
     NEWEST="$( awk '{ print $4 " " $5 }' "$d.${SUFFIX}.lst" | sort | tail -n 1 )"
-    touch \
-      -d "$NEWEST" \
-      "$d.${SUFFIX}" \
-      "$d.${SUFFIX}.lst"
+    if [[ -n "$SPLIT" ]]; then
+      touch \
+        -d "$NEWEST" \
+        "$d.${SUFFIX}".*[0123456789] \
+        "$d.${SUFFIX}.lst"
+    else
+      touch \
+        -d "$NEWEST" \
+        "$d.${SUFFIX}" \
+        "$d.${SUFFIX}.lst"
+    fi
 
     if [ "$KEEP_LIST" -eq 0 ]; then
       rm -f "$d.${SUFFIX}.lst"