try with extra RAM

files/archive-subfolders

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-VERSION="20250214"
+VERSION="20250225"
 
 function helpexit() {
     BS=$( basename "$0" )
@@ -13,6 +13,7 @@ function helpexit() {
     echo "  --nl  Do not store a separate file list."
     echo "  --rm  Remove source folders after archival."
     echo "  --split SIZE  Split archive to SIZE. ex. 500G"
+    echo "  --chunk SIZE  Buffer data in RAM to reduce simultaneous read/write. default: $CHUNKSIZE"
     echo "  folder name: If given, archive only the named folder(s)."
     echo "               If not, for loop over all folders in the current directory."
 
@@ -76,6 +77,7 @@ REMOVE=0
 KEEP_LIST=1
 YES=0
 SPLIT=""
+CHUNKSIZE=100M
 for ((i=1; i<=${#@}; i++)) {
     [[ "${!i}" = "-h" ]] && helpexit
     [[ "${!i}" = "--help" ]] && helpexit
@@ -85,6 +87,7 @@ for ((i=1; i<=${#@}; i++)) {
     [[ "${!i}" = "--nl" ]] && { KEEP_LIST=0; continue; }
     [[ "${!i}" = "--rm" ]] && { REMOVE=1; continue; }
     [[ "${!i}" = "--split" ]] && { i=$(( i + 1 )); SPLIT="${!i}"; continue; }
+    [[ "${!i}" = "--chunk" ]] && { i=$(( i + 1 )); CHUNKSIZE="${!i}"; continue; }
     [[ "${!i}" = "-"* ]] && helpexit
     FOLDERS+=( "${!i%/}" )
     DEPTH=0
@@ -100,6 +103,13 @@ if [[ -n "$SPLIT" ]]; then
      exit 1
   fi
 fi
+if [[ "$CHUNKSIZE" =~ ^[0-9]+$ ]] || [[ "$CHUNKSIZE" =~ ^[0-9]+[KMGTPEZ]$ ]] || [[ "$CHUNKSIZE" =~ ^[0-9]+[KMGTPEZ]B$ ]]; then
+  true
+else
+  echo "Cannot parse: $CHUNKSIZE. use ex. 256M.  valid extension K M G T P E Z"
+  exit 1
+fi
+CHUNKCMD="| dd obs=$CHUNKSIZE status=none "
 if tar --version | grep -q GNU; then
   SORTFILES="--sort=name"
 fi
@@ -134,7 +144,7 @@ for d in "${REALFOLDERS[@]}"; do
   fi
   printf "\n"
   eval "tar cvv $SORTFILES --index-file=\"$d.${SUFFIX}.lst\" \"$d\" | \
-        pv -s \"$SIZE\" $COMPRESSCMD $STORECMD" && {
+        pv -s \"$SIZE\" $COMPRESSCMD $CHUNKCMD $STORECMD" && {
     # tar exists okay
     NEWEST="$( awk '{ print $4 " " $5 }' "$d.${SUFFIX}.lst" | sort | tail -n 1 )"
     if [[ -n "$SPLIT" ]]; then