From 9ef188ff01ea1c93d8ee590def414f0461f79ebd Mon Sep 17 00:00:00 2001 From: ville rantanen Date: Tue, 20 Jan 2015 10:28:21 +0200 Subject: [PATCH] early spring cleaning --- anduril/300_byobu_squeue | 16 -- anduril/Folder2Array | 45 ----- anduril/anduril-remove-files | 105 ----------- anduril/anduril-vm-mail | 21 --- anduril/csc_vm_slurm_query | 19 -- anduril/csc_vm_status_query | 75 -------- anduril/exec_copy.py | 183 ------------------- anduril/slurm | 53 ------ anduril/slurm-cake | 27 --- anduril/slurm-leastalloc | 1 - anduril/slurm-maxfree | 131 ------------- anduril/slurm-random | 126 ------------- anduril/slurp | 1 - anduril/slurpy | 47 ----- anduril/srun-maxfree | 49 ----- anduril/srun-random | 46 ----- anduril/status_query.py | 33 ---- anduril/status_reporter.py | 58 ------ anduril/testing/anduril-build-component-docs | 54 ------ anduril/testing/anduril-test-component | 70 ------- anduril/testing/anduril-test-network | 63 ------- anduril/testing/testcasecopy.sh | 48 ----- anduril/vm_report_status | 44 ----- bin/Folder2Array | 1 - bin/anduril-build-component-docs | 1 - bin/anduril-test-component | 1 - bin/anduril-test-network | 1 - 27 files changed, 1319 deletions(-) delete mode 100755 anduril/300_byobu_squeue delete mode 100755 anduril/Folder2Array delete mode 100755 anduril/anduril-remove-files delete mode 100755 anduril/anduril-vm-mail delete mode 100755 anduril/csc_vm_slurm_query delete mode 100755 anduril/csc_vm_status_query delete mode 100755 anduril/exec_copy.py delete mode 100755 anduril/slurm delete mode 100755 anduril/slurm-cake delete mode 120000 anduril/slurm-leastalloc delete mode 100755 anduril/slurm-maxfree delete mode 100755 anduril/slurm-random delete mode 120000 anduril/slurp delete mode 100755 anduril/slurpy delete mode 100755 anduril/srun-maxfree delete mode 100755 anduril/srun-random delete mode 100755 anduril/status_query.py delete mode 100755 anduril/status_reporter.py delete mode 100755 anduril/testing/anduril-build-component-docs delete mode 100755 anduril/testing/anduril-test-component delete mode 100755 anduril/testing/anduril-test-network delete mode 100755 anduril/testing/testcasecopy.sh delete mode 100755 anduril/vm_report_status delete mode 120000 bin/Folder2Array delete mode 120000 bin/anduril-build-component-docs delete mode 120000 bin/anduril-test-component delete mode 120000 bin/anduril-test-network diff --git a/anduril/300_byobu_squeue b/anduril/300_byobu_squeue deleted file mode 100755 index 888f7da..0000000 --- a/anduril/300_byobu_squeue +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -NODELIST=( $( scontrol show node|grep NodeName | sed -e 's,[^ ]\+=\([^ ]\+\) .*,\1,' ) ) - -NODERUNS=( ) - -# find node with least allocated CPUs - -for e in ${NODELIST[@]} -do NODERUNS+=( $( scontrol show node $e | grep CPUAlloc | sed 's,.*CPUAlloc=\([0-9]\+\).*,\1,' ) ) -done -for (( i=0; i<${#NODERUNS[@]}; i++ )) -do echo -n "$(( $i+3 )):${NODERUNS[$i]} " -done -echo "" - diff --git a/anduril/Folder2Array b/anduril/Folder2Array deleted file mode 100755 index 6cd4e1e..0000000 --- a/anduril/Folder2Array +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -f - -MAXDEPTH="-maxdepth 1" -HIDDEN="-not -path */\.*" -KEYS="awk {gsub(/\//,\"_\",\$1);print}" -ME=$( basename "$0" ) -HELP='Create an array _index for Anduril of the given folder. - -Usage: '$ME' [-anr] [path] - -a Include ".*" hidden files - -n Use number for key - -r Recursively find all files in path, exclude folder names from output - -' - -while getopts ":anrh" opt; do - case $opt in - a) - HIDDEN="" - ;; - n) - KEYS='awk $1=NR' - ;; - r) - MAXDEPTH="-type f" - ;; - h) - echo "$HELP" - exit 0 - ;; - \?) - echo "Invalid option: -$OPTARG" >&2 - echo "$HELP" >&2 - exit 1 - ;; - esac -done -shift $((OPTIND-1)) -INPATH="$1" -[ -z "$INPATH" ] && INPATH=. - -echo -e '"Key"'"\t"'"File"' -find "$INPATH" -mindepth 1 $MAXDEPTH $HIDDEN -not -name _index -printf '"%P"\t"%P"\n' \ - | sort -V | $KEYS diff --git a/anduril/anduril-remove-files b/anduril/anduril-remove-files deleted file mode 100755 index 96f3a25..0000000 --- a/anduril/anduril-remove-files +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/python -import sys -import os,shutil -import re - -from argparse import ArgumentParser - -LOGFILE="_global" -EXECFILE="_state" - -def setup_options(): - parser=ArgumentParser(description="Remove execution folders recusively") - parser.add_argument("-l",action="store_true",dest="logs",default=False, - help="Search for log folders [%(default)s]") - parser.add_argument("-e",action="store_true",dest="execs",default=False, - help="Search for execution folders [%(default)s]") - parser.add_argument("-f",action="store_true",dest="force",default=False, - help="Force deletion, otherwise just lists the entries. WARNING: THIS OPTION DELETES FILES RECURSIVELY WITHOUT ASKING. [%(default)s]") - parser.add_argument("-L",action="store_true",dest="symlinks",default=False, - help="Follow symbolic links [%(default)s]") - parser.add_argument("-m",type=str,action='store',dest="move",default="", - help="Move folders to a given subfolder, instead of deleting. WARNING: MOVES FILES WITHOUT ASKING. Moving is only sensible within the same filesystem.") - parser.add_argument('startpath', action="store",default='.', nargs='?') - - options=parser.parse_args() - if options.move!="": - options.force=False - if not options.move.endswith("/"): - options.move+="/" - if (not options.logs) and (not options.execs): - print("Nothing to do, not printing logs, nor execution folders") - sys.exit(1) - return options - - -def delete_folder(folder): - # Delete folder if it exists - if os.path.exists(folder): - shutil.rmtree(folder) - -def move_folder(folder,options): - # Move a folder to a location, with folder name flattened /->_ - if not os.path.exists(options.move): - os.makedirs(options.move) - new_name=folder - while new_name.startswith(".") or new_name.startswith("/"): - new_name=new_name[1:] - new_name=new_name.replace("/","_") - sys.stdout.write(new_name) - new_name=os.path.join(options.move,new_name) - if os.path.exists(new_name): - sys.stdout.write(" Folder already exists! Not moving.") - return - shutil.move(folder, new_name) - -def recurse(options): - - for path,dirs,files in os.walk(options.startpath,followlinks=options.symlinks): - # do not move files in move folder a second time.. - if options.move!="": - if os.path.exists(options.move) and os.path.samefile(options.move, path): - del dirs[:] - continue - take_action=False - if options.logs: - if LOGFILE in files: - take_action=True - if EXECFILE in files: - del dirs[:] - # never descend in to execution directories - if options.execs: - take_action=True - - if take_action: - del dirs[:] - sys.stdout.write(path) - # only one of the following will be true: - if options.force: - sys.stdout.write(" Deleting") - delete_folder(path) - if options.move!="": - sys.stdout.write(" -> "+options.move) - move_folder(path,options) - sys.stdout.write("\n") - else: - #Recurse with cleaned out dirs - dirs.sort() - dirs=clean_dirs(dirs) - - return - -def clean_dirs(dirs): - # removes hidden folders - for s in dirs[:]: - if (s.startswith(".")): - dirs.remove(s) - return dirs - -def main(): - options=setup_options(); - recurse(options) - sys.exit(0) - -main() - diff --git a/anduril/anduril-vm-mail b/anduril/anduril-vm-mail deleted file mode 100755 index ec7dff9..0000000 --- a/anduril/anduril-vm-mail +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - - -( - echo 'Subject: Execution finished! '$NETWORK_TO_RUN' - ' - echo "Pipeline: $CLEANER_TITLE" - echo -n "Started in: " - pwd - echo "Host: $HOSTNAME" - echo "Execution folder: $RESULTS_IN" - echo "Log folder: $LOG_IN" - [ -f "$LOG_IN"/_global ] && { - echo "===== Log tail =====" - head -n 1 "$LOG_IN"/_global - tail -n 20 "$LOG_IN"/_global - - } - - ) | sendmail -r$USER@mappi.helsinki.fi $USER@mappi.helsinki.fi & - diff --git a/anduril/csc_vm_slurm_query b/anduril/csc_vm_slurm_query deleted file mode 100755 index 1596a03..0000000 --- a/anduril/csc_vm_slurm_query +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -NODELIST=( $( scontrol show node|grep NodeName | sed -e 's,[^ ]\+=\([^ ]\+\) .*,\1,' ) ) - -NODERUNS=( ) - -# find node with least allocated CPUs - -for e in ${NODELIST[@]} -do NODERUNS+=( $( scontrol show node $e | grep CPUAlloc | sed 's,.*CPUAlloc=\([0-9]\+\).*,\1,' ) ) -done -echo -n "CPUAllocated: " -for (( i=0; i<${#NODERUNS[@]}; i++ )) -do scontrol -o show node ${NODELIST[$i]} | grep State=DOWN >/dev/null && NODERUNS[$i]=DN - echo -n "${NODELIST[$i]}:${NODERUNS[$i]} " - -done -echo "" - diff --git a/anduril/csc_vm_status_query b/anduril/csc_vm_status_query deleted file mode 100755 index b08f339..0000000 --- a/anduril/csc_vm_status_query +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -help='CSC Virtual Machine Load Status - - -s Short (default) - -l Long - -c Do not pretty print short output - -b [l|m] return the name of the node with l=least load, m=least used memory - -u [username] return a list of nodes with processes of given user - - L=Load, M=used memory + cached / total, U=users with processes -' - -mode="short" -postprocess=" | sort -V | ncsv -c -i' ' -d' ' " -while getopts "clshu:b:" OPTS; -do case ${OPTS} in - s) - mode="short" - ;; - l) - mode="long" - ;; - c) - postprocess="" - ;; - b) - mode="best" - [ "$OPTARG" = "l" ] && bestcol=3 - [ "$OPTARG" = "m" ] && bestcol=6 - if [ -z "$bestcol" ] - then echo "Argument to -b not recognized" - echo "$help" - exit - fi - ;; - u) - mode="user" - username=$OPTARG - [ -z "$username" ] && exit - ;; - h) - echo "$help" - exit - ;; - ?) - echo "$help" - exit - ;; - - esac -done -if [ "$OPTIND" -gt 1 ] -then shift $(( $OPTIND-1 )) -fi - -status_folder="/mnt/csc-gc5/vm_state" -if [ ! -d "$status_folder" ] -then echo $status_folder not mounted - exit -fi - -if [ "$mode" = "short" ] -then eval "cat $status_folder/*short $postprocess" -fi -if [ "$mode" = "long" ] -then eval "cat $status_folder/*long" -fi -if [ "$mode" = "best" ] -then eval "cat $status_folder/*short" | sed 's,[|/:\+], ,g' | sort -k $bestcol -n | cut -d" " -f2 | head -n 1 -fi -if [ "$mode" = "user" ] -then eval "grep $username $status_folder/*long" | sed -e 's,.*/,,' -e 's,\..*, ,' | tr -d [:cntrl:] - echo "" -fi diff --git a/anduril/exec_copy.py b/anduril/exec_copy.py deleted file mode 100755 index ac28ed9..0000000 --- a/anduril/exec_copy.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/python - -import os -import sys -import re -import csv -import shutil -from argparse import ArgumentParser - -filesearch=re.compile('^_index$') -STATEFILE='_state' -DRYSTATEFILE='_stateDryRun' - -def setup_options(): - usage='''%(prog)s [options] - Use rsync to copy your execution folder to a new place. - e.g. rsync -avP -e ssh user@server:/source/path/ target/path/ - Then give a replacement pair -i and -o for each expected changed absolute path. - (multiple -i and -o are often required) - - Each replace pair is tried, until a file with that replaced name is found. - ''' - - parser=ArgumentParser(usage=usage) - parser.add_argument("-d",type=str,dest="execpath",default=".", - help="Execution folder for anduril (location for _state), default: %(default)s") - parser.add_argument("-i",type=str,dest="inabs",action="append",default=["."], - help="Input absolute path prefix. e.g. /home1/user1/project/") - parser.add_argument("-o",type=str,dest="outabs",action="append",default=["."], - help="Output absolute path prefix. e.g. /home2/user2/different_project/") - parser.add_argument("-a",action="store_false",dest="arrays",default=True, - help="Disable finding _index (array) files. Just modify _state and nothing else." ) - return parser.parse_args() - -def check_options(opts): - if not (os.path.isfile(os.path.join(opts.execpath,STATEFILE))): - print(STATEFILE+' file not found in folder '+opts.execpath) - sys.exit(1) - if len(opts.inabs) is not len(opts.outabs): - print('A matching pair must be found for each -i/-o argument') - sys.exit(1) - if not opts.execpath.endswith('/'): - opts.execpath=opts.execpath+'/' - for i in xrange(len(opts.inabs)): - if not opts.inabs[i].endswith('/'): - opts.inabs[i]=opts.inabs[i]+'/' - for i in xrange(len(opts.outabs)): - if not opts.outabs[i].endswith('/'): - opts.outabs[i]=opts.outabs[i]+'/' - - return opts - -def getpathlist(path): - ''' Returns a list of subfolders ''' - list=os.listdir(path) - paths=[] - for d in list: - if (os.path.isdir(os.path.join(path,d))): - paths.append(d+'/') - return paths - -def getfilelist(path): - ''' Returns a list of files that might require change ''' - list=os.listdir(path) - files=[] - for f in list: - if (filesearch.match(f)) and (os.path.isfile(os.path.join(path,f))): - files.append(f) - return files - -def statefile(opts,path): - print('Parsing _state file') - shutil.copy2(os.path.join(path,STATEFILE), os.path.join(path,STATEFILE+'.bkp')) - statereader=csv.reader(open(os.path.join(path,STATEFILE),'rb'), - delimiter='\t', - doublequote=False, - escapechar='\\', - quoting=csv.QUOTE_NONE) - stateout=[] - for row in statereader: - rowout=row - if row[3].startswith('INPUT '): - newinput=row[3] - rowpstart=row[3].index(' P path=') - rowtstart=row[3].index(' TS in=') - rowpath=row[3][(rowpstart+7):rowtstart] - rowtime=row[3][(rowtstart+7):-1] - # time has a space at the end (and three zeros...) - print('INPUT found: "'+row[3]+'"') - found=False - for i in xrange(len(opts.inabs)): - newpath=rowpath.replace('='+opts.inabs[i],'='+opts.outabs[i]) - if os.path.exists(newpath[1:]): - found=True - newtime=str(int(os.path.getmtime(newpath[1:])))+'000' - newinput=row[3].replace(rowpath,newpath,1).replace(rowtime,newtime,1) - print('NEW INPUT : "'+newinput+'"') - rowout[3]=newinput - break - if not found: - print('WARN: Could not find new INPUT, check your -i and -o arguments') - stateout.append(rowout) - statewriter=csv.writer(open(os.path.join(path,STATEFILE),'wb'), - delimiter='\t', - doublequote=False, - escapechar='\\', - quoting=csv.QUOTE_NONE) - statewriter.writerows(stateout) - - return - -def arrayreplace(path,filelist,opts): - header=['Key','File'] - for f in filelist: - print('Modifying array: '+os.path.join(path,f)) - arrayreader=csv.DictReader(open(os.path.join(path,f),'rb'), - delimiter='\t', - quotechar='"', - quoting=csv.QUOTE_ALL) - arrayout=[] - for row in arrayreader: - rowout=row - if os.path.exists(os.path.join(path,row['File'])): - arrayout.append(rowout) - continue - # File is a relative path, and exists - next iteration. - rowpath=row['File'] - found=False - for i in xrange(len(opts.inabs)): - newpath=rowpath.replace(opts.inabs[i],opts.outabs[i],1) - if os.path.exists(newpath): - found=True - rowout['File']=newpath - break - if not found: - print('WARN: Could not find File '+rowpath+' in '+os.path.join(path,f)+', check your -i and -o arguments') - arrayout.append(rowout) - - writer = csv.DictWriter(open(os.path.join(path,f),'wb'), - header, - delimiter='\t', - quotechar='"', - quoting=csv.QUOTE_MINIMAL) - writer.writerow(dict(zip(header,header))) - writer.writerows(arrayout) - - return - -def statequery(path): - statereader=csv.reader(open(os.path.join(path,DRYSTATEFILE),'rb'), - delimiter='\t', - doublequote=False, - escapechar='\\', - quoting=csv.QUOTE_NONE) - for row in statereader: - if row[1]=='NO': - print('Instance will run: '+row[0]) - else: - print('Instance wont run: '+row[0]) - - return - - - -def traverse(opts,path): - pathlist=getpathlist(path) - filelist=getfilelist(path) - arrayreplace(path,filelist,opts) - for p in pathlist: - traverse(opts,os.path.join(path,p)) - return - -def main(): - opts=setup_options() - opts=check_options(opts) - - if opts.arrays: - traverse(opts,opts.execpath) - statefile(opts,opts.execpath) - return - - -main() diff --git a/anduril/slurm b/anduril/slurm deleted file mode 100755 index f8fc395..0000000 --- a/anduril/slurm +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -USAGE="Usage: "$( basename $0 )" [options] [--] [command including srun options] -Sends the command to a queue with srun -Version (c) Ville.Rantanen@helsinki.fi -Options: - --possess Possess the whole node for the run - -c N Request number of CPUs [Def 4] - --mem M Request M Gb of memory [Def 20] - -h This help - -Use the -- switch to separate switches assigned to slurp vs anything after.. -" -if [ -z "$1" ] -then echo -e "$USAGE" - exit 1 -fi -SHARE=-s -CPU=4 -MEM=20 -declare -a argv=("$@") -for (( i=1; i<=${#} ;i++ )) -do [[ ${!i} == "--possess" ]] && { - echo Will possess the whole node. - SHARE="" - unset argv[$(( $i - 1 ))] - } - [[ ${!i} == "-c" ]] && { - CPU=${argv[$i]} - unset argv[$(( $i - 1 ))] - unset argv[$(( $i ))] - } - [[ ${!i} == "--mem" ]] && { - MEM=${argv[$i]} - unset argv[$(( $i - 1 ))] - unset argv[$(( $i ))] - continue - } - [[ ${!i} == "--mem="* ]] && { - MEM=${argv[$(( $i - 1 ))]#*=} - unset argv[$(( $i - 1 ))] - } - [[ ${!i} == "-h" ]] && { - echo -e "$USAGE" - exit 1 - } - [[ ${!i} == "--" ]] && { - break - } -done -MEM=$(( 1024*$MEM )) -echo Running command: srun $SHARE -c $CPU --mem $MEM "${argv[@]}" -srun -v $SHARE -c $CPU --mem $MEM "${argv[@]}" diff --git a/anduril/slurm-cake b/anduril/slurm-cake deleted file mode 100755 index bf52e8f..0000000 --- a/anduril/slurm-cake +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -[ -z "$2" ] && { echo 'Usage: $0 "file" "some" "command" - Runs the command when file exists. waits for max 5 minutes' - exit -} - -fname="$1" -shift 1 -otime=1 -while : -do [ -e "$fname" ] || { - echo -n "File: $fname not found! " - date - } - [ -e "$fname" ] && { - break - } - otime=$(( $otime + 1 )) - sleep 1 - [ $otime -gt 300 ] && { - echo "File $fname still missing after 5 minutes!" - exit 1 - } -done - -eval "$@" diff --git a/anduril/slurm-leastalloc b/anduril/slurm-leastalloc deleted file mode 120000 index db9c124..0000000 --- a/anduril/slurm-leastalloc +++ /dev/null @@ -1 +0,0 @@ -slurm-maxfree \ No newline at end of file diff --git a/anduril/slurm-maxfree b/anduril/slurm-maxfree deleted file mode 100755 index dc1f04e..0000000 --- a/anduril/slurm-maxfree +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash - -if [ -z "$1" ] -then echo provide the script to run - exit 1 -fi - -if [ -z "${ANDURIL_NODELIST}" ] -then NODELIST=( $( scontrol show node|grep NodeName | sed -e 's,[^ ]\+=\([^ ]\+\) .*,\1,' ) ) -else read -a NODELIST <<< "$ANDURIL_NODELIST" -fi -JOBROOT="$HOME/.srun" - -if [ "$1" == "-h" ] -then echo -ne ' -This prefix selects the highest free cpu node for slurm run. -Use it with anduril: --exec-mode prefix --prefix '$( basename $0 )' -To change the list of nodes (preferred order): -export ANDURIL_NODELIST="vm3 vm4 vm5" - -Current nodelist: "'${NODELIST[@]}'"'"( ${#NODELIST[@]} nodes)\n" - exit -fi - -NODERUNS=( ) -for e in ${NODELIST[@]} -do NODERUNS+=( 0 ) -done - -# find node with max free CPUs -while [ true ] -do - MAX=0 - INDEX=0 - for (( i=0; i<${#NODERUNS[@]}; i++ )) - do NODERUNS[$i]=$(( $( scontrol -o show node ${NODELIST[$i]} | sed 's,.*CPUAlloc=\([0-9]\+\).*CPUTot=\([0-9]\+\).*,\2-\1 ,' ) )) - scontrol -o show node ${NODELIST[$i]} | grep State=DOWN > /dev/null && NODERUNS[$i]=-1 - echo -n "${NODELIST[$i]}:${NODERUNS[$i]} " - [ ${NODERUNS[$i]} -gt $MAX ] && { - MAX=${NODERUNS[$i]} - INDEX=$i - } - done - [ $MAX -gt 0 ] && { - break - } || { - SECONDS=$(( 10 + $RANDOM/1000 )) - echo "waiting for free sockets for $SECONDS s." - sleep $SECONDS - } -done - -echo "srun: Node ${NODELIST[$INDEX]}, Free sockets: ${NODERUNS[$INDEX]}" >&2 - -mkdir -p "$JOBROOT" -for (( i=1; i<=$#; i++ )) -do if [[ "${!i}" == */_command ]] - then JOBNAME=job_ - COMPONENTNAME=$( grep ^metadata.componentName= "${!i}" | sed s,^metadata.componentName=,, | sed -e 's/[^A-Za-z0-9._-]/_/g' ) - INSTANCENAME=$( grep ^metadata.instanceName= "${!i}" | sed s,^metadata.instanceName=,, | sed -e 's/[^A-Za-z0-9._-]/_/g' ) - JOBNAME="$JOBNAME"$( grep ^metadata.sourceLocation= "${!i}" | sed s,^metadata.sourceLocation=,, | sed -e 's/[^A-Za-z0-9._-]/_/g' ) - JOBNAME="$JOBNAME"_$INSTANCENAME - JOBNAME="$JOBNAME"_$COMPONENTNAME - JOBNAME="$JOBNAME"_$( date +"%y%m%d_%H%M%S" ) - CPU=$( grep ^metadata.cpu= "${!i}" | sed s,^metadata.cpu=,, ) - MEMORY=$( grep ^metadata.memory= "${!i}" | sed s,^metadata.cpu=,, ) - export USERDEFINED=$( grep ^metadata.userDefined= "${!i}" | sed s,^metadata.userDefined=,, ) - fi -done -JOBPATH="$JOBROOT/$JOBNAME" -while [ -d "$JOBPATH" ] -do echo Jobpath "$JOBPATH" exists - JOBNAME=job_$( date +"%y%m%d_%H%M%S" )_$( echo $@ | md5sum | cut -f1 -d" " ) - JOBPATH="$JOBROOT/$JOBNAME" -done -mkdir -p "$JOBPATH" -JOBFILE="$JOBPATH/job" -STATFILE="$JOBPATH/statistics" -STRMFILE="$JOBPATH/stream" -EXECPATH=$( pwd ) -[[ -z "$CPU" ]] || export CPU="-c $CPU" -[[ -z "$MEMORY" ]] || export MEMORY="--mem $MEMORY" - -# create the jobfile -echo '#!/bin/bash' > "$JOBFILE" -chmod 755 "$JOBFILE" -# Find _command file -for (( i=1; i<=$#; i++ )) -do if [[ "${!i}" == */_command ]] - then echo 'retrys=0' >> "$JOBFILE" - echo 'while : - do [ -f "'${!i}'" ] && { - break - } || { - echo Waiting for _command file '${!i}' - retrys=$(( $retrys + 1 )) - sleep 1; [ "$retrys" -gt 15 ] && break; - } - done ' >> "$JOBFILE" - fi -done - -echo 'echo Node: $HOSTNAME'" >> \"$STATFILE\" " >> "$JOBFILE" -echo "pwd >> \"$STATFILE\" " >> "$JOBFILE" -echo "date +'Start: %s' >> \"$STATFILE\" " >> "$JOBFILE" -echo -n "/usr/bin/time -o \"$STATFILE\" --append " >> "$JOBFILE" -for (( i=1; i<=$#; i++ )) -do echo -n "\"${!i}\" " >> "$JOBFILE" -done -#Catch the input stream (for R launcher) -cat - >> "$STRMFILE" -echo -n ' < "'$STRMFILE'"' >> "$JOBFILE" -echo -e "\n" >> "$JOBFILE" -echo 'EC=$?' >> "$JOBFILE" -echo "date +'Stop: %s' >> \"$STATFILE\" " >> "$JOBFILE" -echo 'exit $EC' >> "$JOBFILE" - -echo "The job file is in $JOBFILE" -# send the job -echo \#srun $CPU $MEMORY -J $INSTANCENAME --nodelist=${NODELIST[$INDEX]} slurm-cake "$JOBFILE" "$JOBFILE" >> "$JOBFILE" -srun $CPU $MEMORY -J $INSTANCENAME --nodelist=${NODELIST[$INDEX]} slurm-cake "$JOBFILE" "$JOBFILE" -EC=$? -# clear out old job definitions (semirandomly) -[[ -e "$JOBROOT"/.lastdel ]] || touch "$JOBROOT"/.lastdel -if test "$( find $JOBROOT/.lastdel -mmin +30 )" -then touch "$JOBROOT"/.lastdel - find "$JOBROOT" -maxdepth 2 -mindepth 2 -type f -mtime +20 -delete 2>/dev/null - find "$JOBROOT" -type d -depth -empty -delete 2>/dev/null -fi - -exit $EC diff --git a/anduril/slurm-random b/anduril/slurm-random deleted file mode 100755 index 72a7f89..0000000 --- a/anduril/slurm-random +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -if [ -z "$1" ] -then echo provide the script to run - exit 1 -fi - -if [ -z "${ANDURIL_NODELIST}" ] -then NODELIST=( $( scontrol show node|grep NodeName | sed -e 's,[^ ]\+=\([^ ]\+\) .*,\1,' ) ) -else read -a NODELIST <<< "$ANDURIL_NODELIST" -fi -JOBROOT="$HOME/.srun" - -if [ "$1" == "-h" ] -then echo -ne ' -This tool selects a random node for slurm run. -Use it with anduril: --exec-mode prefix --prefix '$( basename $0 )' -To change the list of nodes for randomization: -export ANDURIL_NODELIST="vm3 vm4 vm5" - -Current nodelist: "'${NODELIST[@]}'"'"( ${#NODELIST[@]} nodes)\n" - exit -fi - -NODERUNS=( ) -TRY=0 -for e in ${NODELIST[@]} -do NODERUNS+=( 0 ) -done -while [ true ] -do - INDEX=$(( ( RANDOM % ${#NODELIST[@]} ) )) - NODERUNS=$(( $( scontrol -o show node ${NODELIST[$INDEX]} | sed 's,.*CPUAlloc=\([0-9]\+\).*CPUTot=\([0-9]\+\).*,\2-\1 ,' ) )) - scontrol -o show node ${NODELIST[$INDEX]} | grep State=DOWN > /dev/null && NODERUNS=-1 - echo "srun: Node ${NODELIST[$INDEX]}, Free sockets: $NODERUNS" >&2 - [ $NODERUNS -gt 0 ] && { - break - } || { - TRY=$(( $TRY + 1 )) - echo "try again $TRY" - [ $TRY -gt 3 ] && { - TRY=0 - SECONDS=$(( ( RANDOM % 10 ) )) - echo "waiting for free sockets for $SECONDS s." - sleep $SECONDS - } - } -done - -mkdir -p "$JOBROOT" -for (( i=1; i<=$#; i++ )) -do if [[ "${!i}" == */_command ]] - then JOBNAME=job_ - COMPONENTNAME=$( grep ^metadata.componentName= "${!i}" | sed s,^metadata.componentName=,, | sed -e 's/[^A-Za-z0-9._-]/_/g' ) - INSTANCENAME=$( grep ^metadata.instanceName= "${!i}" | sed s,^metadata.instanceName=,, | sed -e 's/[^A-Za-z0-9._-]/_/g' ) - JOBNAME="$JOBNAME"$( grep ^metadata.sourceLocation= "${!i}" | sed s,^metadata.sourceLocation=,, | sed -e 's/[^A-Za-z0-9._-]/_/g' ) - JOBNAME="$JOBNAME"_$INSTANCENAME - JOBNAME="$JOBNAME"_$COMPONENTNAME - JOBNAME="$JOBNAME"_$( date +"%y%m%d_%H%M%S" ) - CPU=$( grep ^metadata.cpu= "${!i}" | sed s,^metadata.cpu=,, ) - MEMORY=$( grep ^metadata.memory= "${!i}" | sed s,^metadata.cpu=,, ) - export USERDEFINED=$( grep ^metadata.userDefined= "${!i}" | sed s,^metadata.userDefined=,, ) - fi -done -JOBPATH="$JOBROOT/$JOBNAME" -while [ -d "$JOBPATH" ] -do echo Jobpath "$JOBPATH" exists - JOBNAME=job_$( date +"%y%m%d_%H%M%S" )_$( echo $@ | md5sum | cut -f1 -d" " ) - JOBPATH="$JOBROOT/$JOBNAME" -done -mkdir -p "$JOBPATH" -JOBFILE="$JOBPATH/job" -STATFILE="$JOBPATH/statistics" -STRMFILE="$JOBPATH/stream" -EXECPATH=$( pwd ) -[[ -z "$CPU" ]] || export CPU="-c $CPU" -[[ -z "$MEMORY" ]] || export MEMORY="--mem $MEMORY" - -# create the jobfile -echo '#!/bin/bash' > "$JOBFILE" -chmod 755 "$JOBFILE" -# Find _command file -for (( i=1; i<=$#; i++ )) -do if [[ "${!i}" == */_command ]] - then echo 'retrys=0' >> "$JOBFILE" - echo 'while : - do [ -f "'${!i}'" ] && { - break - } || { - echo Waiting for _command file '${!i}' - retrys=$(( $retrys + 1 )) - sleep 1; [ "$retrys" -gt 15 ] && break; - } - done ' >> "$JOBFILE" - fi -done - -echo 'echo Node: $HOSTNAME'" >> \"$STATFILE\" " >> "$JOBFILE" -echo "pwd >> \"$STATFILE\" " >> "$JOBFILE" -echo "date +'Start: %s' >> \"$STATFILE\" " >> "$JOBFILE" -echo -n "/usr/bin/time -o \"$STATFILE\" --append " >> "$JOBFILE" -for (( i=1; i<=$#; i++ )) -do echo -n "\"${!i}\" " >> "$JOBFILE" -done -#Catch the input stream (for R launcher) -cat - >> "$STRMFILE" -echo -n ' < "'$STRMFILE'"' >> "$JOBFILE" -echo -e "\n" >> "$JOBFILE" -echo 'EC=$?' >> "$JOBFILE" -echo "date +'Stop: %s' >> \"$STATFILE\" " >> "$JOBFILE" -echo 'exit $EC' >> "$JOBFILE" - -echo "The job file is in $JOBFILE" -# send the job -echo \#srun $CPU $MEMORY -J $INSTANCENAME --nodelist=${NODELIST[$INDEX]} slurm-cake "$JOBFILE" "$JOBFILE" >> "$JOBFILE" -srun $CPU $MEMORY -J $INSTANCENAME --nodelist=${NODELIST[$INDEX]} slurm-cake "$JOBFILE" "$JOBFILE" -EC=$? -# clear out old job definitions (semirandomly) -[[ -e "$JOBROOT"/.lastdel ]] || touch "$JOBROOT"/.lastdel -if test "$( find $JOBROOT/.lastdel -mmin +30 )" -then touch "$JOBROOT"/.lastdel - find "$JOBROOT" -maxdepth 2 -mindepth 2 -type f -mtime +20 -delete 2>/dev/null - find "$JOBROOT" -type d -depth -empty -delete 2>/dev/null -fi - -exit $EC diff --git a/anduril/slurp b/anduril/slurp deleted file mode 120000 index 0efb82a..0000000 --- a/anduril/slurp +++ /dev/null @@ -1 +0,0 @@ -slurm \ No newline at end of file diff --git a/anduril/slurpy b/anduril/slurpy deleted file mode 100755 index abfd213..0000000 --- a/anduril/slurpy +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -if [ -z "$1" ] -then echo provide the script to run - exit 1 -fi - -JOBPATH="$( pwd )/.slurpies" -mkdir -p "$JOBPATH" -JOBNAME=job_$( date +"%y%m%d%H%M%S%N" ) -JOBFILE="$JOBPATH/$JOBNAME" -DONEFILE="$JOBPATH/$JOBNAME.done" -COMPPATH=$3 - -echo Job: $JOBFILE -echo "#!/bin/sh" > "$JOBFILE" -for (( i=1; i<=$#; i++ )) -do if [ $i -eq 5 ] - then echo -n \" >> "$JOBFILE" - eval echo -n ${!i} | sed s,\",\\\\\",g >> "$JOBFILE" - echo -n "\" " >> "$JOBFILE" - elif [ $i -eq 6 ] - then echo -n ${!i} >> "$JOBFILE" - else echo -n "\"${!i}\" " >> "$JOBFILE" - fi -done -echo "" >> "$JOBFILE" -echo 'find "'$COMPPATH'" > "'$DONEFILE'"' >> "$JOBFILE" -chmod 775 "$JOBFILE" -slurp "$JOBFILE" -sleep 1 -# pause to make sure NFS has time to sync -while [ ! -e "$DONEFILE" ] -do sleep 5 -done -DONE=0 -IFS=$'\n' -while [ $DONE -eq 0 ] -do sleep 10 - DONEITER=1 - for f in $( cat "$DONEFILE" ) - do [[ -a "$f" ]] || DONEITER=0 - done - [[ $DONEITER -eq 1 ]] && DONE=1 -done - - diff --git a/anduril/srun-maxfree b/anduril/srun-maxfree deleted file mode 100755 index 083ac25..0000000 --- a/anduril/srun-maxfree +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -if [ -z "${ANDURIL_NODELIST}" ] -then NODELIST=( $( scontrol show node|grep NodeName | sed -e 's,[^ ]\+=\([^ ]\+\) .*,\1,' ) ) -else read -a NODELIST <<< "$ANDURIL_NODELIST" -fi - -if [ "$1" == "-h" ] -then echo -ne ' -This tools selects the lowest cpu allocated node for slurm run -export ANDURIL_NODELIST="vm3 vm4 vm5" - ^ to have your own preferred list of nodes -Current nodelist: "'${NODELIST[@]}'"'"( ${#NODELIST[@]} nodes)\n" - exit -fi - - -NODERUNS=( ) -# find node with max free CPUs -for e in ${NODELIST[@]} -do NODERUNS+=( 0 ) -done -while [ true ] -do - MAX=0 - INDEX=0 - for (( i=0; i<${#NODERUNS[@]}; i++ )) - do NODERUNS[$i]=$(( $( scontrol -o show node ${NODELIST[$i]} | sed 's,.*CPUAlloc=\([0-9]\+\).*CPUTot=\([0-9]\+\).*,\2-\1 ,' ) )) - scontrol -o show node ${NODELIST[$i]} | grep State=DOWN > /dev/null && NODERUNS[$i]=-1 - echo -n "${NODELIST[$i]}:${NODERUNS[$i]} " - [ ${NODERUNS[$i]} -gt $MAX ] && { - MAX=${NODERUNS[$i]} - INDEX=$i - } - done - [ $MAX -gt 0 ] && { - break - } || { - SECONDS=$(( ( RANDOM % 30 ) )) - echo "waiting for free sockets for $SECONDS s." - sleep $SECONDS - } -done - -echo "srun: Node ${NODELIST[$INDEX]}, Free sockets: ${NODERUNS[$INDEX]}" >&2 - -srun --nodelist=${NODELIST[$INDEX]} "$@" - - diff --git a/anduril/srun-random b/anduril/srun-random deleted file mode 100755 index 370a02f..0000000 --- a/anduril/srun-random +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -if [ -z "${ANDURIL_NODELIST}" ] -then NODELIST=( $( scontrol show node|grep NodeName | sed -e 's,[^ ]\+=\([^ ]\+\) .*,\1,' ) ) -else read -a NODELIST <<< "$ANDURIL_NODELIST" -fi - -if [ "$1" == "-h" ] -then echo -ne ' -This tools selects the lowest cpu allocated node for slurm run -export ANDURIL_NODELIST="vm3 vm4 vm5" - ^ to have your own preferred list of nodes -Current nodelist: "'${NODELIST[@]}'"'"( ${#NODELIST[@]} nodes)\n" - exit -fi - -NODERUNS=( ) -TRY=0 -# find node with max free CPUs -for e in ${NODELIST[@]} -do NODERUNS+=( 0 ) -done -while [ true ] -do - INDEX=$(( ( RANDOM % ${#NODELIST[@]} ) )) - NODERUNS=$(( $( scontrol -o show node ${NODELIST[$INDEX]} | sed 's,.*CPUAlloc=\([0-9]\+\).*CPUTot=\([0-9]\+\).*,\2-\1 ,' ) )) - scontrol -o show node ${NODELIST[$INDEX]} | grep State=DOWN > /dev/null && NODERUNS=-1 - echo "srun: Node ${NODELIST[$INDEX]}, Free sockets: $NODERUNS" >&2 - [ $NODERUNS -gt 0 ] && { - break - } || { - TRY=$(( $TRY + 1 )) - echo "try again $TRY" - [ $TRY -gt 3 ] && { - TRY=0 - SECONDS=$(( ( RANDOM % 10 ) )) - echo "waiting for free sockets for $SECONDS s." - sleep $SECONDS - } - } -done - - -srun --nodelist=${NODELIST[$INDEX]} "$@" - - diff --git a/anduril/status_query.py b/anduril/status_query.py deleted file mode 100755 index 47c2b06..0000000 --- a/anduril/status_query.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/python -#echo client program -import socket -import sys - -if len(sys.argv)<2: - message='HELO' -else: - message=sys.argv[1] - - -hosts=['vm1', - 'vm2', - 'vm3', - 'vm4', - 'vm5', - 'vm6', - 'narsil'] - -PORT = 50774 # The same port as used by the server - -for HOST in hosts: - try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect((HOST, PORT)) - s.settimeout(10) - s.send(message) - data = s.recv(1024) - sys.stdout.write(data) - s.close() - except: - sys.stdout.write('|'+HOST+' does not answer. ') - diff --git a/anduril/status_reporter.py b/anduril/status_reporter.py deleted file mode 100755 index 6e9a34e..0000000 --- a/anduril/status_reporter.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/python -#echo server program -import socket -import subprocess - -help_message="""Recognized commands: -HELO (default) -SHORT - -""" - -HOST = '' # Symbolic name meaning all available interfaces -PORT = 50774 # Arbitrary non-privileged port -s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -s.bind((HOST, PORT)) -while 1: - s.listen(1) - conn, addr = s.accept() - #print 'Connected by', addr - while 1: - data = conn.recv(1024) - if not data: break - message="" - if (data=="HELO"): - uptime=subprocess.Popen(['uptime'], stdout=subprocess.PIPE) - mem=subprocess.Popen(['free -tg | head -n 2'], stdout=subprocess.PIPE,shell=True) - users=subprocess.Popen(['who -q | head -n 1'], stdout=subprocess.PIPE,shell=True) - message='='*10+socket.gethostname()+'='*10+'\n'+uptime.stdout.read()+mem.stdout.read()+users.stdout.read() - uptime.kill() - mem.kill() - users.kill() - elif (data=="SHORT"): - load=open('/proc/loadavg','r').read() - load=load.split(" ") - cpu=open('/proc/cpuinfo','r').read() - cpucount=0 - for row in cpu.split("\n"): - if row.startswith('processor'): - cpucount+=1 - loadpercent=int(100*float(load[0])/float(cpucount)) - - mem=open('/proc/meminfo','r').read() - memory={'total':0, 'free':0, 'cache':0, 'percent':0} - for row in mem.split("\n"): - if row.startswith('MemTotal'): - memory['total']=float(''.join(c for c in row if c.isdigit())) - if row.startswith('MemFree'): - memory['free']=float(''.join(c for c in row if c.isdigit())) - if row.startswith('Cached'): - memory['cache']=float(''.join(c for c in row if c.isdigit())) - memory['percent']=int(100 - (100*(memory['free']+memory['cache'])/memory['total'])) - - message='|'+socket.gethostname()+' L/M:'+str(loadpercent).zfill(3)+'/'+str(memory['percent']).zfill(3)+'% ' - else: - message=help_message - conn.send(message) - conn.close() - diff --git a/anduril/testing/anduril-build-component-docs b/anduril/testing/anduril-build-component-docs deleted file mode 100755 index 1cd0b61..0000000 --- a/anduril/testing/anduril-build-component-docs +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -function return_error { - echo "$1" - exit 1 -} - -ANDURIL_ENV=$HOME/.config/anduril/environment.sh - -if [ -f "$ANDURIL_ENV" ] -then . "$ANDURIL_ENV" - echo Read "$ANDURIL_ENV" -fi - -[ -d "$ANDURIL_HOME" ] || return_error "ANDURIL_HOME is not set! You may set it also in file: $ANDURIL_ENV" -[ -d "$ANDURIL_BUNDLES" ] || return_error "ANDURIL_BUNDLES is not set! (folder of bundles, may also be same as ANDURIL_HOME)" - -echo ANDURIL_HOME="$ANDURIL_HOME" -echo ANDURIL_BUNDLES="$ANDURIL_BUNDLES" - -if [ ! -z "$1" ] -then search=$( echo $1 | tr A-Z a-z ) -fi - -MICROHOME="$ANDURIL_HOME/microarray" -bundles=( ) -echo -n "Looking for bundles: " -for e in $( ls $ANDURIL_BUNDLES ) -do [[ -e ${ANDURIL_BUNDLES}/$e/bundle.xml ]] || continue - bname=$( cat ${ANDURIL_BUNDLES}/$e/bundle.xml | tr -d [:cntrl:] | grep '' | sed 's/.*\(.*\)<\/name>.*/\1/' | tr A-Z a-z ) - bundles+=( $bname ) - echo -n "$bname " - if [ "$search" = "$bname" ] - then BUNDLESTRING="-b ${ANDURIL_BUNDLES}/$e" - fi - if [ "$1" = "all" ] - then BUNDLESTRING="-b ${ANDURIL_BUNDLES}/$e $BUNDLESTRING" - fi - if [ "$bname" = "microarray" ] - then MICROHOME="${ANDURIL_BUNDLES}/$e" - fi -done -echo "" - -if [ -z "$1" ] -then return_error "Give bundle name to create component docs! 'all' matches to all bundles in ANDURIL_BUNDLES" -fi - -[ -d "$MICROHOME" ] || return_error "Could not find Microarray bundle home: $MICROHOME" -[ -z "$BUNDLESTRING" ] && return_error "Could not find given bundle: $search" - - -echo $ANDURIL_HOME/bin/anduril build-doc result_doc --log log_doc "$BUNDLESTRING" -t $MICROHOME/datatypes.xml -eval $ANDURIL_HOME/bin/anduril build-doc result_doc --log log_doc "$BUNDLESTRING" -t $MICROHOME/datatypes.xml diff --git a/anduril/testing/anduril-test-component b/anduril/testing/anduril-test-component deleted file mode 100755 index 16e386b..0000000 --- a/anduril/testing/anduril-test-component +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -function return_error { - echo "$1" - exit 1 -} - -ANDURIL_ENV=$HOME/.config/anduril/environment.sh - -if [ -f "$ANDURIL_ENV" ] -then . "$ANDURIL_ENV" - echo Read "$ANDURIL_ENV" -fi - -[ -d "$ANDURIL_HOME" ] || return_error "ANDURIL_HOME is not set! You may set it also in file: $ANDURIL_ENV" -[ -d "$ANDURIL_BUNDLES" ] || return_error "ANDURIL_BUNDLES is not set! (folder of bundles, may also be same as ANDURIL_HOME)" - -echo ANDURIL_HOME="$ANDURIL_HOME" -echo ANDURIL_BUNDLES="$ANDURIL_BUNDLES" - -echo -n "Looking for bundles: " - b="$ANDURIL_HOME/builtin/bundle.xml" - BUNDLESTRING=" -b $( readlink -f $( dirname ${b} ) )" - bname=$( cat "$b" | tr -d [:cntrl:] | grep '' | sed 's/.*\(.*\)<\/name>.*/\1/' ) - echo -n "$bname " - [ -d "$( dirname ${b} )/components" ] && \ - list=$( find $( dirname ${b} )/components -maxdepth 2 -mindepth 2 -name component.xml -printf '%P\\n' | sed 's,/component.xml,,g' ) - [ -d "$( dirname ${b} )/functions" ] && \ - list=${list}$( find $( dirname ${b} )/functions -maxdepth 2 -mindepth 2 -name component.xml -printf '%P\\n' | sed 's,/component.xml,,g' ) - -for b in $( find -L "$ANDURIL_BUNDLES" -maxdepth 2 -name bundle.xml | grep -v '/testsystem/|/techtest/' ); do - bname=$( cat "$b" | tr -d [:cntrl:] | grep '' | sed 's/.*\(.*\)<\/name>.*/\1/' ) - echo -n "$bname " - [ -d "$( dirname ${b} )/components" ] && \ - list=${list}$( find $( dirname ${b} )/components -maxdepth 2 -mindepth 2 -name component.xml -printf '%P\\n' | sed 's,/component.xml,,g' ) - [ -d "$( dirname ${b} )/functions" ] && \ - list=${list}$( find $( dirname ${b} )/functions -maxdepth 2 -mindepth 2 -name component.xml -printf '%P\\n' | sed 's,/component.xml,,g' ) - BUNDLESTRING=${BUNDLESTRING}" -b $( dirname ${b} )" -done -echo "" -list=( $( echo -e $list | sort )) - -if [ -z "$1" ] -then - for (( n=0; n < ${#list[@]} ; n++ )) - do echo $n: ${list[$n]} - done - echo "all: ALL" - echo "clean: delete log and exec folders" - exit 0 -fi -if [[ "$1" == ?([0-9]*) ]] -then # is numeric - echo test ${list[$1]} - echo $ANDURIL_HOME/bin/anduril test ${list[$1]} --log log_component -d result_component ${BUNDLESTRING} - $ANDURIL_HOME/bin/anduril test ${list[$1]} --log log_component -d result_component ${BUNDLESTRING} -else # is not numeric - if [ $1 = "all" ] - then for (( n=0; n < ${#list[@]} ; n++ )) - do echo running: ${list[$n]} - $ANDURIL_HOME/bin/anduril test ${list[$n]} --log log_component -d result_component ${BUNDLESTRING} - done - elif [ $1 = "clean" ] - then rm -vrf log_component result_component - else - echo test $1 - echo $ANDURIL_HOME/bin/anduril test $1 --log log_component -d result_component ${BUNDLESTRING} - $ANDURIL_HOME/bin/anduril test $1 --log log_component -d result_component ${BUNDLESTRING} - fi -fi diff --git a/anduril/testing/anduril-test-network b/anduril/testing/anduril-test-network deleted file mode 100755 index 6c1102c..0000000 --- a/anduril/testing/anduril-test-network +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -function return_error { - echo "$1" - exit 1 -} - -ANDURIL_ENV=$HOME/.config/anduril/environment.sh - -if [ -f "$ANDURIL_ENV" ] -then . "$ANDURIL_ENV" - echo Read "$ANDURIL_ENV" -fi - -[ -d "$ANDURIL_HOME" ] || return_error "ANDURIL_HOME is not set! You may set it also in file: $ANDURIL_ENV" -[ -d "$ANDURIL_BUNDLES" ] || return_error "ANDURIL_BUNDLES is not set! (folder of bundles, may also be same as ANDURIL_HOME)" - - -echo -n "Looking for bundles: " - b="$ANDURIL_HOME/builtin/bundle.xml" - BUNDLESTRING=" -b $( readlink -f $( dirname ${b} ) )" - bname=$( cat "$b" | tr -d [:cntrl:] | grep '' | sed 's/.*\(.*\)<\/name>.*/\1/' ) - echo -n "$bname " - [ -d "$( dirname ${b} )/test-networks" ] && \ - list=$( find $( dirname ${b} )/test-networks -maxdepth 2 -mindepth 2 -name network.and -printf '%P\\n' | sed 's,/network.and,,g' ) - -for b in $( find -L "$ANDURIL_BUNDLES" -maxdepth 2 -name bundle.xml | grep -v testsystem ); do - BUNDLESTRING=${BUNDLESTRING}" -b $( readlink -f $( dirname ${b} ) )" - bname=$( cat "$b" | tr -d [:cntrl:] | grep '' | sed 's/.*\(.*\)<\/name>.*/\1/' ) - echo -n "$bname " - [ -d "$( dirname ${b} )/test-networks" ] && \ - list=${list}$( find $( dirname ${b} )/test-networks -maxdepth 2 -mindepth 2 -name network.and -printf '%P\\n' | sed 's,/network.and,,g' ) -done -echo "" -list=( $( echo -e $list | sort )) - -if [ -z "$1" ] -then - for (( n=0; n < ${#list[@]} ; n++ )) - do echo $n: ${list[$n]} - done - echo "all: ALL" - echo "clean: delete log and exec folders" - exit 0 -fi -if [[ "$1" == ?([0-9]*) ]] -then # is numeric - echo test ${list[$1]} - echo $ANDURIL_HOME/bin/anduril test-networks ${list[$1]} --log log_network -d result_network ${BUNDLESTRING} - $ANDURIL_HOME/bin/anduril test-networks ${list[$1]} --log log_network -d result_network ${BUNDLESTRING} -else # is not numeric - if [ $1 = "all" ] - then for (( n=0; n < ${#list[@]} ; n++ )) - do echo running: ${list[$n]} - $ANDURIL_HOME/bin/anduril test-networks ${list[$n]} --log log_network -d result_network ${BUNDLESTRING} - done - elif [ $1 = "clean" ] - then rm -vrf log_network result_network - else - echo test $1 - $ANDURIL_HOME/bin/anduril test-networks $1 --log log_network -d result_network ${BUNDLESTRING} - fi -fi diff --git a/anduril/testing/testcasecopy.sh b/anduril/testing/testcasecopy.sh deleted file mode 100755 index 1f25045..0000000 --- a/anduril/testing/testcasecopy.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -if [ -z "$2" ] -then - echo " -Copies Anduril testcase results to the original source location. -Use it when testcases fail, but you know the results to be correct. -After copying you must commit the changes in the checkout. - -Usage: testcasecopy executionfolder/ComponentFolder /path/to/bundle/root - -example: testcasecopy results/CSVJoin ~/anduril/trunk/microarray/ - -Any extra arguments are given to the copy command: cp -a [args] source target -Recommended use: -i for prompt before overwrite and -v for verbose -" -exit 0 -fi -IFS=$'\n' -component=$( basename "$1" ) -execfolder=$( dirname "$1" ) -source=$1 -bundle=$2 -shift 2 -if [ ! -d "$source" ] -then echo "$source folder not found" - exit 1 -fi -if [ ! -d "$bundle" ] -then echo "$bundle folder not found" - exit 1 -fi -#echo $execfolder $component -cases=$( find "$source" -mindepth 1 -maxdepth 1 -type d ) -for case in $cases -do - casebase=$( basename "$case" ) - tgt="${bundle}/components/${component}/testcases/${casebase}/expected-output" - if [ -d "$tgt" ] - then - for output in $( ls "${case}/component" | grep -v ^_ ) - do - cp -a $@ "${case}/component/${output}" "${tgt}/" - done - else - echo "$component $casebase does not have expected-output folder" - fi -done \ No newline at end of file diff --git a/anduril/vm_report_status b/anduril/vm_report_status deleted file mode 100755 index b6573c6..0000000 --- a/anduril/vm_report_status +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -cores=$( grep -c processor /proc/cpuinfo ) - -# verbose mode -tgtf=/mnt/csc-gc5/vm_state/${HOSTNAME}.long -# debug -if [ -z "$1" ] -then touch "$tgtf" -fi -if [ -f "$tgtf" ] -then echo "======== $HOSTNAME =======" > "$tgtf" - TZ="Europe/Helsinki" uptime >> "$tgtf" - echo "Cores: $cores" >> "$tgtf" - free -tg | head -n 2 >> "$tgtf" - ps a --format user| grep -v -e root -e USER | sort -u | tr '\n' ' ' >> "$tgtf" - echo "" >> "$tgtf" -fi - -# concise mode - -tgtf=/mnt/csc-gc5/vm_state/${HOSTNAME}.short -# debug -if [ -z "$1" ] -then touch "$tgtf" -fi -if [ -f "$tgtf" ] -then echo -n "|$HOSTNAME L:" > "$tgtf" - load=$( cat /proc/loadavg | cut -d" " -f3 | tr -d [:cntrl:] ) - echo -n "$load/$cores M:" >> "$tgtf" - # used + cached = total - free - free=$(( $( grep ^MemFree /proc/meminfo | tr -d -c [:digit:] ) / 1048576 )) - cache=$(( $( grep ^Cached /proc/meminfo | tr -d -c [:digit:] ) / 1048576 )) - total=$(( $( grep ^MemTotal /proc/meminfo | tr -d -c [:digit:] ) / 1048576 )) - used=$(( $total -$free -$cache )) - echo -n "$used+$cache/$total U:" >> "$tgtf" - ps a --format user| grep -v -e root -e USER | sort -u | wc -l | tr -c -d [:digit:] >> "$tgtf" - TZ="Europe/Helsinki" date "+ @%H:%M" >> "$tgtf" -# save history - echo -e "$HOSTNAME\t$load\t$used\t"$( date +%s ) >> /mnt/csc-gc5/vm_state/history/${HOSTNAME} -fi - - - diff --git a/bin/Folder2Array b/bin/Folder2Array deleted file mode 120000 index 1a341ae..0000000 --- a/bin/Folder2Array +++ /dev/null @@ -1 +0,0 @@ -../anduril/Folder2Array \ No newline at end of file diff --git a/bin/anduril-build-component-docs b/bin/anduril-build-component-docs deleted file mode 120000 index a5eafa1..0000000 --- a/bin/anduril-build-component-docs +++ /dev/null @@ -1 +0,0 @@ -../anduril/testing/anduril-build-component-docs \ No newline at end of file diff --git a/bin/anduril-test-component b/bin/anduril-test-component deleted file mode 120000 index 2b52e3a..0000000 --- a/bin/anduril-test-component +++ /dev/null @@ -1 +0,0 @@ -../anduril/testing/anduril-test-component \ No newline at end of file diff --git a/bin/anduril-test-network b/bin/anduril-test-network deleted file mode 120000 index 2f47c03..0000000 --- a/bin/anduril-test-network +++ /dev/null @@ -1 +0,0 @@ -../anduril/testing/anduril-test-network \ No newline at end of file