diff --git a/anduril/csc_vm_status_query b/anduril/csc_vm_status_query new file mode 100755 index 0000000..802347a --- /dev/null +++ b/anduril/csc_vm_status_query @@ -0,0 +1,75 @@ +#!/bin/bash + +help='CSC Virtual Machine Load Status + + -s Short (default) + -l Long + -c Do not pretty print short output + -b [l|m] return the name of the node with l=least load, m=least used memory + -u [username] return a list of nodes with processes of given user + + L=Load, M=used memory + cached / total, U=users with processes +' + +mode="short" +postprocess=" | ncsv -c -i' ' -d' ' " +while getopts "clshu:b:" OPTS; +do case ${OPTS} in + s) + mode="short" + ;; + l) + mode="long" + ;; + c) + postprocess="" + ;; + b) + mode="best" + [ "$OPTARG" = "l" ] && bestcol=3 + [ "$OPTARG" = "m" ] && bestcol=6 + if [ -z "$bestcol" ] + then echo "Argument to -b not recognized" + echo "$help" + exit + fi + ;; + u) + mode="user" + username=$OPTARG + [ -z "$username" ] && exit + ;; + h) + echo "$help" + exit + ;; + ?) + echo "$help" + exit + ;; + + esac +done +if [ "$OPTIND" -gt 1 ] +then shift $(( $OPTIND-1 )) +fi + +status_folder="/mnt/csc-cloud/vm_state" +if [ ! -d "$status_folder" ] +then echo $status_folder not mounted + exit +fi + +if [ "$mode" = "short" ] +then eval "cat $status_folder/*short $postprocess" +fi +if [ "$mode" = "long" ] +then eval "cat $status_folder/*long" +fi +if [ "$mode" = "best" ] +then eval "cat $status_folder/*short" | sed 's,[|/:\+], ,g' | sort -k $bestcol -n | cut -d" " -f2 | head -n 1 +fi +if [ "$mode" = "user" ] +then eval "grep $username $status_folder/*long" | sed -e 's,.*/,,' -e 's,\..*, ,' | tr -d [:cntrl:] + echo "" +fi diff --git a/anduril/exec_copy.py b/anduril/exec_copy.py new file mode 100755 index 0000000..f82f9c3 --- /dev/null +++ b/anduril/exec_copy.py @@ -0,0 +1,191 @@ +#!/usr/bin/python + +import os +import sys +import re +import csv +import shutil +from argparse import ArgumentParser + +filesearch=re.compile('^_index$') +STATEFILE='_state' +DRYSTATEFILE='_stateDryRun' + +def setup_options(): + usage='''%(prog)s [options] + Use rsync to copy your execution folder to a new place. + e.g. rsync -avP -e ssh user@server:/source/path/ target/path/ + Then give a replacement pair -i and -o for each expected changed absolute path. + (multiple -i and -o are often required) + + Each replace pair is tried, until a file with that replaced name is found. + ''' + + parser=ArgumentParser(usage=usage) + parser.add_argument("-d",type=str,dest="execpath",default=".", + help="Execution folder for anduril (location for _state), default: %(default)s") + parser.add_argument("-i",type=str,dest="inabs",action="append",default=["."], + help="Input absolute path prefix. e.g. /home1/user1/project/") + parser.add_argument("-o",type=str,dest="outabs",action="append",default=["."], + help="Output absolute path prefix. e.g. /home2/user2/different_project/") + parser.add_argument("-q",action="store_true",dest="query",default=False, + help="Compare "+STATEFILE+" and "+DRYSTATEFILE+" to "+ + "see which components are not up-to-date. Use the anduril command: "+ + "anduril run --dry ... to create the dry state file" ) + return parser.parse_args() + +def check_options(opts): + if not (os.path.isfile(os.path.join(opts.execpath,STATEFILE))): + print(STATEFILE+' file not found in folder '+opts.execpath) + sys.exit(1) + if opts.query: + if not (os.path.isfile(os.path.join(opts.execpath,DRYSTATEFILE))): + print(DRYSTATEFILE+' file not found in folder '+opts.execpath) + sys.exit(1) + if len(opts.inabs) is not len(opts.outabs): + print('A matching pair must be found for each -i/-o argument') + sys.exit(1) + if not opts.execpath.endswith('/'): + opts.execpath=opts.execpath+'/' + for i in xrange(len(opts.inabs)): + if not opts.inabs[i].endswith('/'): + opts.inabs[i]=opts.inabs[i]+'/' + for i in xrange(len(opts.outabs)): + if not opts.outabs[i].endswith('/'): + opts.outabs[i]=opts.outabs[i]+'/' + + return opts + +def getpathlist(path): + ''' Returns a list of subfolders ''' + list=os.listdir(path) + paths=[] + for d in list: + if (os.path.isdir(os.path.join(path,d))): + paths.append(d+'/') + return paths + +def getfilelist(path): + ''' Returns a list of files that might require change ''' + list=os.listdir(path) + files=[] + for f in list: + if (filesearch.match(f)) and (os.path.isfile(os.path.join(path,f))): + files.append(f) + return files + +def statefile(opts,path): + print('Parsing _state file') + shutil.copy2(os.path.join(path,STATEFILE), os.path.join(path,STATEFILE+'.bkp')) + statereader=csv.reader(open(os.path.join(path,STATEFILE),'rb'), + delimiter='\t', + doublequote=False, + escapechar='\\', + quoting=csv.QUOTE_NONE) + stateout=[] + for row in statereader: + rowout=row + if row[3].startswith('INPUT '): + newinput=row[3] + rowpstart=row[3].index(' P path=') + rowtstart=row[3].index(' TS in=') + rowpath=row[3][(rowpstart+7):rowtstart] + rowtime=row[3][(rowtstart+7):-1] + # time has a space at the end (and three zeros...) + print('INPUT found: "'+row[3]+'"') + found=False + for i in xrange(len(opts.inabs)): + newpath=rowpath.replace('='+opts.inabs[i],'='+opts.outabs[i]) + if os.path.exists(newpath[1:]): + found=True + newtime=str(int(os.path.getmtime(newpath[1:])))+'000' + newinput=row[3].replace(rowpath,newpath,1).replace(rowtime,newtime,1) + print('NEW INPUT : "'+newinput+'"') + rowout[3]=newinput + break + if not found: + print('WARN: Could not find new INPUT, check your -i and -o arguments') + stateout.append(rowout) + statewriter=csv.writer(open(os.path.join(path,STATEFILE),'wb'), + delimiter='\t', + doublequote=False, + escapechar='\\', + quoting=csv.QUOTE_NONE) + statewriter.writerows(stateout) + + return + +def arrayreplace(path,filelist,opts): + header=['Key','File'] + for f in filelist: + print('Modifying array: '+os.path.join(path,f)) + arrayreader=csv.DictReader(open(os.path.join(path,f),'rb'), + delimiter='\t', + quotechar='"', + quoting=csv.QUOTE_ALL) + arrayout=[] + for row in arrayreader: + rowout=row + if os.path.exists(os.path.join(path,row['File'])): + arrayout.append(rowout) + continue + # File is a relative path, and exists - next iteration. + rowpath=row['File'] + found=False + for i in xrange(len(opts.inabs)): + newpath=rowpath.replace(opts.inabs[i],opts.outabs[i],1) + if os.path.exists(newpath): + found=True + rowout['File']=newpath + break + if not found: + print('WARN: Could not find File '+rowpath+' in '+os.path.join(path,f)+', check your -i and -o arguments') + arrayout.append(rowout) + + writer = csv.DictWriter(open(os.path.join(path,f),'wb'), + header, + delimiter='\t', + quotechar='"', + quoting=csv.QUOTE_MINIMAL) + writer.writerow(dict(zip(header,header))) + writer.writerows(arrayout) + + return + +def statequery(path): + statereader=csv.reader(open(os.path.join(path,DRYSTATEFILE),'rb'), + delimiter='\t', + doublequote=False, + escapechar='\\', + quoting=csv.QUOTE_NONE) + for row in statereader: + if row[1]=='NO': + print('Instance will run: '+row[0]) + else: + print('Instance wont run: '+row[0]) + + return + + + +def traverse(opts,path): + pathlist=getpathlist(path) + filelist=getfilelist(path) + arrayreplace(path,filelist,opts) + for p in pathlist: + traverse(opts,os.path.join(path,p)) + return + +def main(): + opts=setup_options() + opts=check_options(opts) + + if opts.query: + statequery(opts.execpath) + else: + traverse(opts,opts.execpath) + statefile(opts,opts.execpath) + return + + +main() diff --git a/anduril/rownicer.py b/anduril/rownicer.py new file mode 100755 index 0000000..dd1f91e --- /dev/null +++ b/anduril/rownicer.py @@ -0,0 +1,48 @@ +#!/usr/bin/python +import sys +import math + +if (len(sys.argv)<2): + print('''Anduril network file indentation tool: + give the filename to break... + + ''') + sys.exit() + + +f=open(sys.argv[1],'r') +orig=f.read() +size=len(orig) +f.close() +chrcount=0 +brace=0 +firstofrow=True +inquotes=False +space=' ' + +for i in range(size): + c=orig[i] + chrcount+=1 + if c=="\t": + sys.stdout.write(" ") + else: + sys.stdout.write(c) + if c=='"': + inquotes = not inquotes + if ( not inquotes ): + if c=="\n": + chrcount=0 + brace=0 + firstofrow=True + if ( c=="(" ) & ( firstofrow ): + brace=int(min(32,math.ceil((chrcount)/4)*4)) + firstofrow=False + if ( i+1 "$tgtf" + TZ="Europe/Helsinki" uptime >> "$tgtf" + echo "Cores: $cores" >> "$tgtf" + free -tg | head -n 2 >> "$tgtf" + ps a --format user| grep -v -e root -e USER | sort -u | tr '\n' ' ' >> "$tgtf" + echo "" >> "$tgtf" +fi + +# concise mode + +tgtf=/mnt/csc-cloud/vm_state/${HOSTNAME}.short +# debug +if [ -z "$1" ] +then touch "$tgtf" +fi +if [ -f "$tgtf" ] +then echo -n "|$HOSTNAME L:" > "$tgtf" + load=$( cat /proc/loadavg | cut -d" " -f3 | tr -d [:cntrl:] ) + echo -n "$load/$cores M:" >> "$tgtf" + # used + cached = total - free + free=$(( $( grep ^MemFree /proc/meminfo | tr -d -c [:digit:] ) / 1048576 )) + cache=$(( $( grep ^Cached /proc/meminfo | tr -d -c [:digit:] ) / 1048576 )) + total=$(( $( grep ^MemTotal /proc/meminfo | tr -d -c [:digit:] ) / 1048576 )) + used=$(( $total -$free -$cache )) + echo -n "$used+$cache/$total U:" >> "$tgtf" + ps a --format user| grep -v -e root -e USER | sort -u | wc -l | tr -c -d [:digit:] >> "$tgtf" + TZ="Europe/Helsinki" date "+ @%H:%M" >> "$tgtf" +# save history + echo -e "$HOSTNAME\t$load\t$used\t"$( date +%s ) >> /mnt/csc-cloud/vm_state/history/${HOSTNAME} +fi + + + diff --git a/highbeam b/highbeam new file mode 100755 index 0000000..0b2a88e --- /dev/null +++ b/highbeam @@ -0,0 +1,73 @@ +#!/bin/bash + +function usage { +echo ' High Beam Highlighter +- a trial to create a very generic modifyable +syntax highlighter. + +Example rules: ( ~/.highbeamrc ) +-------------------- +RULES=( + "[0-9]" "$C" # Show numbers with Cyan color. NOTE: since the colors are represented with numbers, having this later in the rules will break coloring! + "$USER[ ]\+$USER" "$BGC$BLACK" # Use quotes to have rules with spaces. This colors the "ls -la" usernames. background cyan, black text + status "$G" # simple word matching, coloring green + "^d[^ ]*" "$H$G" # color words at the beginning of line, starting with d. color bright green + "q.\{0,3\}r" "$H$Y" # q*r with maximum 3 characters in between. color bright yellow +) +-------------------- +Color variables: $R $G $B $Y $M $C $W $BLACK +Modify to bright version by prefixing with $H (e.g. $H$G) +Background colors: $BGR $BGG $BGB $BGY $BGC $BGM $BGW +' + +} + +[[ "$1" = "-h" ]] && usage && exit + +E='\x1b[' +H="${E}1m" +Z="${E}0m" + +# list of text colors: +BLACK="${E}30m" +R="${E}31m" +G="${E}32m" +B="${E}34m" +Y="${E}33m" +M="${E}35m" +C="${E}36m" +W="${E}37m" + + +# list of BG colors: +BGR="${E}41m" +BGG="${E}42m" +BGB="${E}44m" +BGY="${E}43m" +BGM="${E}45m" +BGC="${E}46m" +BGW="${E}47m" + +# user rules: +[[ -e ~/.highbeamrc ]] && . ~/.highbeamrc || echo You may have wrong construction in .highbeamrc >&2 || exit +[[ -e ~/.highbeamrc ]] || { +# list of rules +RULES=( + "[0-9]" "$C" # Show numbers with Cyan color. NOTE: since the colors are represented with numbers, having this later in the rules will break coloring! + "$USER[ ]\+$USER" "$BGC$BLACK" # Use quotes to have rules with spaces. background cyan, black text + status "$G" # simple word matching, coloring green + "^d[^ ]*" "$H$G" # color words at the beginning of line, starting with d. color bright green + "q.\{0,3\}r" "$H$Y" # q*r with maximum 3 characters in between. color bright yellow +) +} +# actual code starts here + +for (( r=0; r<${#RULES[@]}; r++ )); +do REGEX="$REGEX -e 's/\(${RULES[$r]}\)/${RULES[$(( $r + 1 ))]}\1${Z}/ig'" + r=$(( $r + 1 )) +done +eval sed $REGEX || echo Maybe error in .highbeamrc >&2 + + + + diff --git a/vncdisp b/vnc/vncdisp similarity index 100% rename from vncdisp rename to vnc/vncdisp diff --git a/vnckill b/vnc/vnckill similarity index 100% rename from vnckill rename to vnc/vnckill diff --git a/vncs b/vnc/vncs similarity index 100% rename from vncs rename to vnc/vncs