merge

anduril/slurm-maxfree

@@ -23,21 +23,32 @@ Current nodelist: "'${NODELIST[@]}'"'"( ${#NODELIST[@]} nodes)\n"
 fi
 
 NODERUNS=( )
+for e in ${NODELIST[@]}
+do NODERUNS+=( 0 )
+done
 
 # find node with max free CPUs
-
-for e in ${NODELIST[@]}
-do NODERUNS+=( $(( $( scontrol show node $e | tr -d [:cntrl:] | sed 's,.*CPUAlloc=\([0-9]\+\).*CPUTot=\([0-9]\+\).*,\2-\1 ,' ) )) )
-done
-MAX=${NODERUNS[0]}
-INDEX=0
-for (( i=0; i<${#NODERUNS[@]}; i++ ))
-do (( ${NODERUNS[$i]} > MAX )) && {
+while [ true ]
+do
+    MAX=0
+    INDEX=0
+    for (( i=0; i<${#NODERUNS[@]}; i++ ))
+    do  NODERUNS[$i]=$(( $( scontrol -o show node ${NODELIST[$i]} | sed 's,.*CPUAlloc=\([0-9]\+\).*CPUTot=\([0-9]\+\).*,\2-\1 ,' ) ))
+        echo -n "${NODELIST[$i]}:${NODERUNS[$i]} "
+        [ ${NODERUNS[$i]} -gt $MAX ] && {
             MAX=${NODERUNS[$i]}
             INDEX=$i
         }    
-   echo -n "${NODELIST[$i]}:${NODERUNS[$i]} "
+    done
+    [ $MAX -gt 0 ] && {
+        break
+    } || {
+        SECONDS=$(( 10 + $RANDOM/1000 ))
+        echo "waiting for free sockets for $SECONDS s."
+        sleep $SECONDS
+    }
 done
+
 echo "srun: Node ${NODELIST[$INDEX]}, Free sockets: ${NODERUNS[$INDEX]}" >&2
 
 mkdir -p "$JOBROOT"