moving projects

anduril/csc_vm_status_query

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+help='CSC Virtual Machine Load Status
+
+ -s Short (default)
+ -l Long 
+ -c Do not pretty print short output
+ -b [l|m] return the name of the node with l=least load, m=least used memory
+ -u [username]  return a list of nodes with processes of given user
+
+  L=Load,  M=used memory + cached / total,  U=users with processes
+'
+
+mode="short"
+postprocess=" | ncsv -c -i' ' -d' ' "
+while getopts "clshu:b:" OPTS;
+do case ${OPTS} in
+    s) 
+        mode="short"
+        ;;
+    l) 
+        mode="long"
+        ;;
+    c)
+        postprocess=""
+        ;;
+    b)
+        mode="best"
+        [ "$OPTARG" = "l" ] && bestcol=3
+        [ "$OPTARG" = "m" ] && bestcol=6
+        if [ -z "$bestcol" ]
+        then echo "Argument to -b not recognized"
+             echo "$help"
+             exit
+        fi
+        ;;
+    u)
+        mode="user"
+        username=$OPTARG
+        [ -z "$username" ] && exit
+        ;;
+    h)    
+        echo "$help"
+        exit
+        ;;
+    ?)
+        echo "$help"
+        exit
+        ;;
+
+   esac
+done
+if [ "$OPTIND" -gt 1 ]
+then shift $(( $OPTIND-1 ))
+fi
+
+status_folder="/mnt/csc-cloud/vm_state"
+if [ ! -d "$status_folder" ]
+then echo $status_folder not mounted
+     exit
+fi
+
+if [ "$mode" = "short" ]
+then eval "cat $status_folder/*short $postprocess"
+fi
+if [ "$mode" = "long" ]
+then eval "cat $status_folder/*long"
+fi
+if [ "$mode" = "best" ]
+then eval "cat $status_folder/*short" | sed 's,[|/:\+], ,g' | sort -k $bestcol -n | cut -d" " -f2 | head -n 1
+fi
+if [ "$mode" = "user" ]
+then eval "grep $username $status_folder/*long" | sed -e 's,.*/,,' -e 's,\..*, ,' | tr -d [:cntrl:]
+     echo "" 
+fi

anduril/exec_copy.py

@@ -0,0 +1,191 @@
+#!/usr/bin/python
+
+import os
+import sys
+import re
+import csv
+import shutil
+from argparse import ArgumentParser
+
+filesearch=re.compile('^_index$')
+STATEFILE='_state'
+DRYSTATEFILE='_stateDryRun'
+
+def setup_options():
+    usage='''%(prog)s [options]
+    Use rsync to copy your execution folder to a new place. 
+    e.g.  rsync -avP -e ssh user@server:/source/path/ target/path/
+    Then give a replacement pair -i and -o for each expected changed absolute path.
+    (multiple -i and -o are often required)
+    
+    Each replace pair is tried, until a file with that replaced name is found.
+    '''
+
+    parser=ArgumentParser(usage=usage)
+    parser.add_argument("-d",type=str,dest="execpath",default=".",
+                     help="Execution folder for anduril (location for _state), default: %(default)s")
+    parser.add_argument("-i",type=str,dest="inabs",action="append",default=["."],
+                     help="Input absolute path prefix. e.g. /home1/user1/project/")
+    parser.add_argument("-o",type=str,dest="outabs",action="append",default=["."],
+                     help="Output absolute path prefix. e.g. /home2/user2/different_project/")
+    parser.add_argument("-q",action="store_true",dest="query",default=False,
+                     help="Compare "+STATEFILE+" and "+DRYSTATEFILE+" to "+
+                          "see which components are not up-to-date. Use the anduril command: "+
+                          "anduril run --dry ... to create the dry state file" )
+    return parser.parse_args()
+    
+def check_options(opts):
+    if not (os.path.isfile(os.path.join(opts.execpath,STATEFILE))):
+        print(STATEFILE+' file not found in folder '+opts.execpath)
+        sys.exit(1)
+    if opts.query:
+        if not (os.path.isfile(os.path.join(opts.execpath,DRYSTATEFILE))):
+            print(DRYSTATEFILE+' file not found in folder '+opts.execpath)
+            sys.exit(1)        
+    if len(opts.inabs) is not len(opts.outabs):
+        print('A matching pair must be found for each -i/-o argument')
+        sys.exit(1)
+    if not opts.execpath.endswith('/'):
+         opts.execpath=opts.execpath+'/'
+    for i in xrange(len(opts.inabs)):
+        if not opts.inabs[i].endswith('/'):
+            opts.inabs[i]=opts.inabs[i]+'/'
+    for i in xrange(len(opts.outabs)):
+        if not opts.outabs[i].endswith('/'):
+            opts.outabs[i]=opts.outabs[i]+'/'
+           
+    return opts
+    
+def getpathlist(path):
+    ''' Returns a list of subfolders '''
+    list=os.listdir(path)
+    paths=[]
+    for d in list:
+        if (os.path.isdir(os.path.join(path,d))):
+            paths.append(d+'/')
+    return paths  
+
+def getfilelist(path):
+    ''' Returns a list of files that might require change '''
+    list=os.listdir(path)
+    files=[]
+    for f in list:
+        if (filesearch.match(f)) and (os.path.isfile(os.path.join(path,f))):
+            files.append(f)
+    return files    
+
+def statefile(opts,path):
+    print('Parsing _state file')
+    shutil.copy2(os.path.join(path,STATEFILE), os.path.join(path,STATEFILE+'.bkp'))
+    statereader=csv.reader(open(os.path.join(path,STATEFILE),'rb'),
+                        delimiter='\t',
+                        doublequote=False,
+                        escapechar='\\',
+                        quoting=csv.QUOTE_NONE)
+    stateout=[]
+    for row in statereader:
+        rowout=row
+        if row[3].startswith('INPUT '):
+            newinput=row[3]
+            rowpstart=row[3].index(' P path=')
+            rowtstart=row[3].index(' TS in=')
+            rowpath=row[3][(rowpstart+7):rowtstart]
+            rowtime=row[3][(rowtstart+7):-1]
+            # time has a space at the end (and three zeros...)
+            print('INPUT found: "'+row[3]+'"')
+            found=False
+            for i in xrange(len(opts.inabs)):
+                newpath=rowpath.replace('='+opts.inabs[i],'='+opts.outabs[i])
+                if os.path.exists(newpath[1:]):
+                    found=True
+                    newtime=str(int(os.path.getmtime(newpath[1:])))+'000'
+                    newinput=row[3].replace(rowpath,newpath,1).replace(rowtime,newtime,1)
+                    print('NEW INPUT  : "'+newinput+'"')
+                    rowout[3]=newinput
+                    break
+            if not found:
+                print('WARN: Could not find new INPUT, check your -i and -o arguments')
+        stateout.append(rowout)    
+    statewriter=csv.writer(open(os.path.join(path,STATEFILE),'wb'),
+                        delimiter='\t',
+                        doublequote=False,
+                        escapechar='\\',
+                        quoting=csv.QUOTE_NONE)
+    statewriter.writerows(stateout)
+        
+    return
+
+def arrayreplace(path,filelist,opts):
+    header=['Key','File']
+    for f in filelist:
+        print('Modifying array: '+os.path.join(path,f))
+        arrayreader=csv.DictReader(open(os.path.join(path,f),'rb'),
+                               delimiter='\t',
+                               quotechar='"', 
+                               quoting=csv.QUOTE_ALL)
+        arrayout=[]
+        for row in arrayreader:
+            rowout=row
+            if os.path.exists(os.path.join(path,row['File'])):
+                arrayout.append(rowout)
+                continue
+                # File is a relative path, and exists - next iteration.
+            rowpath=row['File']
+            found=False
+            for i in xrange(len(opts.inabs)):
+                newpath=rowpath.replace(opts.inabs[i],opts.outabs[i],1)
+                if os.path.exists(newpath):
+                    found=True
+                    rowout['File']=newpath
+                    break
+            if not found:
+                print('WARN: Could not find File '+rowpath+' in '+os.path.join(path,f)+', check your -i and -o arguments')
+            arrayout.append(rowout)
+        
+        writer = csv.DictWriter(open(os.path.join(path,f),'wb'),
+                            header,
+                            delimiter='\t',
+                            quotechar='"',
+                            quoting=csv.QUOTE_MINIMAL)
+        writer.writerow(dict(zip(header,header)))
+        writer.writerows(arrayout)
+        
+    return
+    
+def statequery(path):
+    statereader=csv.reader(open(os.path.join(path,DRYSTATEFILE),'rb'),
+                        delimiter='\t',
+                        doublequote=False,
+                        escapechar='\\',
+                        quoting=csv.QUOTE_NONE)
+    for row in statereader:
+        if row[1]=='NO':
+            print('Instance will run: '+row[0])
+        else:
+            print('Instance wont run: '+row[0])
+   
+    return
+    
+        
+    
+def traverse(opts,path):
+    pathlist=getpathlist(path)
+    filelist=getfilelist(path)
+    arrayreplace(path,filelist,opts)
+    for p in pathlist:
+        traverse(opts,os.path.join(path,p))
+    return
+    
+def main():
+    opts=setup_options()
+    opts=check_options(opts)
+
+    if opts.query:
+        statequery(opts.execpath)
+    else:
+        traverse(opts,opts.execpath)
+        statefile(opts,opts.execpath)
+    return
+
+
+main()

anduril/rownicer.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python
+import sys
+import math
+
+if (len(sys.argv)<2):
+   print('''Anduril network file indentation tool:
+   give the filename to break...
+
+   ''')
+   sys.exit()
+
+
+f=open(sys.argv[1],'r')
+orig=f.read()
+size=len(orig)
+f.close()
+chrcount=0
+brace=0
+firstofrow=True
+inquotes=False
+space=' '
+
+for i in range(size):
+    c=orig[i]
+    chrcount+=1
+    if c=="\t":
+        sys.stdout.write("    ")
+    else:
+        sys.stdout.write(c)
+    if c=='"':
+        inquotes = not inquotes
+    if ( not inquotes ):
+        if c=="\n":
+            chrcount=0
+            brace=0
+            firstofrow=True
+        if ( c=="(" ) & ( firstofrow ):
+            brace=int(min(32,math.ceil((chrcount)/4)*4))
+            firstofrow=False
+        if ( i+1<size ):
+            if ( c=="," ):
+                if ( orig[i+1] == " " ):
+                    spacecount=1
+                    while ( orig[i+spacecount] == " " ):
+                        spacecount+=1
+                    sys.stdout.write('\n' + (1+brace-spacecount)*' ')
+                elif ( orig[i+1] != "\n"):
+                    sys.stdout.write('\n' + (brace)*' ')

anduril/slurp

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+USAGE="Usage: "$( basename $0 )" [options] [--] [command including srun options]
+Sends the command to a queue with srun
+Version (c) Ville.Rantanen@helsinki.fi
+Options:
+    --possess Possess the whole node for the run
+    -c N      Request number of CPUs [Def 4]
+    --mem M   Request M Gb of memory [Def 20]
+    -h This help
+
+Use the -- switch to separate switches assigned to slurp vs anything after..
+"
+if [ -z "$1" ]
+then echo -e "$USAGE"
+     exit 1
+fi
+SHARE=-s
+CPU=4
+MEM=20
+declare -a argv=("$@")
+for (( i=1; i<=${#} ;i++ ))
+do [[ ${!i} == "--possess" ]] && {
+       echo Will possess the whole node.
+       SHARE=""
+       unset argv[$(( $i - 1 ))]
+   }
+   [[ ${!i} == "-c" ]] && {
+      CPU=${argv[$i]}
+      unset argv[$(( $i - 1 ))]
+      unset argv[$(( $i ))]
+   }
+   [[ ${!i} == "--mem" ]] && {
+      MEM=${argv[$i]}
+      unset argv[$(( $i - 1 ))]
+      unset argv[$(( $i ))]
+   }
+   [[ ${!i} == "-h" ]] && {
+      echo -e "$USAGE"
+      exit 1
+   }
+  [[ ${!i} == "--" ]] && {
+      break
+  }
+done
+
+echo Running command: srun $SHARE -c $CPU --mem $MEM "${argv[@]}"
+srun -v $SHARE -c $CPU --mem $MEM "${argv[@]}"

anduril/slurpy

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+if [ -z "$1" ]
+then echo provide the script to run
+     exit 1
+fi
+
+JOBPATH="$( pwd )/.slurpies"
+mkdir -p "$JOBPATH"
+JOBNAME=job_$( date +"%y%m%d%H%M%S%N" )
+JOBFILE="$JOBPATH/$JOBNAME"
+DONEFILE="$JOBPATH/$JOBNAME.done"
+COMPPATH=$3
+
+echo Job: $JOBFILE
+echo "#!/bin/sh" > "$JOBFILE"
+for (( i=1; i<=$#; i++ ))
+do if [ $i -eq 5 ]
+   then echo -n \" >> "$JOBFILE" 
+        eval echo -n ${!i} | sed s,\",\\\\\",g  >> "$JOBFILE"
+        echo -n "\" " >> "$JOBFILE"
+   elif [ $i -eq 6 ]
+   then echo -n ${!i} >> "$JOBFILE" 
+   else echo -n "\"${!i}\" " >> "$JOBFILE"
+   fi
+done
+echo "" >> "$JOBFILE"
+echo 'find "'$COMPPATH'" > "'$DONEFILE'"' >> "$JOBFILE"
+chmod 775 "$JOBFILE"
+slurp "$JOBFILE"
+sleep 1
+# pause to make sure NFS has time to sync
+while [ ! -e "$DONEFILE" ]
+do sleep 5
+done
+DONE=0
+IFS=$'\n'
+while [ $DONE -eq 0 ]
+do  sleep 10
+    DONEITER=1
+    for f in $( cat "$DONEFILE" ) 
+    do [[ -a "$f" ]] || DONEITER=0
+    done
+    [[ $DONEITER -eq 1 ]] && DONE=1
+done
+
+

anduril/status_query.py

@@ -0,0 +1,33 @@
+#!/usr/bin/python
+#echo client program
+import socket
+import sys
+
+if len(sys.argv)<2:
+    message='HELO'
+else:
+    message=sys.argv[1]
+
+
+hosts=['vm1',
+       'vm2',
+       'vm3',
+       'vm4',
+       'vm5',
+       'vm6',
+       'narsil']
+
+PORT = 50774              # The same port as used by the server
+
+for HOST in hosts:
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.connect((HOST, PORT))
+        s.settimeout(10)
+        s.send(message)
+        data = s.recv(1024)
+        sys.stdout.write(data)
+        s.close()
+    except:
+        sys.stdout.write('|'+HOST+' does not answer. ')
+

anduril/status_reporter.py

@@ -0,0 +1,58 @@
+#!/usr/bin/python
+#echo server program
+import socket
+import subprocess
+
+help_message="""Recognized commands:
+HELO (default)
+SHORT
+
+"""
+
+HOST = ''                 # Symbolic name meaning all available interfaces
+PORT = 50774              # Arbitrary non-privileged port
+s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+s.bind((HOST, PORT))
+while 1:
+    s.listen(1)
+    conn, addr = s.accept()
+    #print 'Connected by', addr
+    while 1:
+        data = conn.recv(1024)
+        if not data: break
+        message=""
+        if (data=="HELO"):
+            uptime=subprocess.Popen(['uptime'], stdout=subprocess.PIPE)
+            mem=subprocess.Popen(['free -tg | head -n 2'], stdout=subprocess.PIPE,shell=True)
+            users=subprocess.Popen(['who -q | head -n 1'], stdout=subprocess.PIPE,shell=True)
+            message='='*10+socket.gethostname()+'='*10+'\n'+uptime.stdout.read()+mem.stdout.read()+users.stdout.read()
+            uptime.kill()
+            mem.kill()
+            users.kill()
+        elif (data=="SHORT"):
+            load=open('/proc/loadavg','r').read()
+            load=load.split(" ")
+            cpu=open('/proc/cpuinfo','r').read()
+            cpucount=0
+            for row in cpu.split("\n"):
+                if row.startswith('processor'):
+                    cpucount+=1
+            loadpercent=int(100*float(load[0])/float(cpucount))
+
+            mem=open('/proc/meminfo','r').read()
+            memory={'total':0, 'free':0, 'cache':0, 'percent':0}
+            for row in mem.split("\n"):
+                if row.startswith('MemTotal'):
+                    memory['total']=float(''.join(c for c in row if c.isdigit()))
+                if row.startswith('MemFree'):
+                    memory['free']=float(''.join(c for c in row if c.isdigit()))
+                if row.startswith('Cached'):
+                    memory['cache']=float(''.join(c for c in row if c.isdigit()))
+            memory['percent']=int(100 - (100*(memory['free']+memory['cache'])/memory['total']))
+
+            message='|'+socket.gethostname()+' L/M:'+str(loadpercent).zfill(3)+'/'+str(memory['percent']).zfill(3)+'% '
+        else:
+            message=help_message
+        conn.send(message)
+    conn.close()
+

anduril/testcasecopy.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+if [ -z "$2" ]
+then
+    echo "
+Copies Anduril testcase results to the original source location.
+Use it when testcases fail, but you know the results to be correct.
+After copying you must commit the changes in the checkout.
+
+Usage: testcasecopy executionfolder/ComponentFolder /path/to/bundle/root
+
+example: testcasecopy results/CSVJoin ~/anduril/trunk/microarray/
+
+Any extra arguments are given to the copy command: cp -a [args] source target
+Recommended use: -i for prompt before overwrite and -v for verbose
+"
+exit 0
+fi
+IFS=$'\n'
+component=$( basename "$1" )
+execfolder=$( dirname "$1" )
+source=$1
+bundle=$2
+shift 2
+if [ ! -d "$source" ]
+then echo "$source folder not found"
+    exit 1
+fi
+if [ ! -d "$bundle" ]
+then echo "$bundle folder not found"
+    exit 1
+fi
+#echo $execfolder $component
+cases=$( find "$source" -mindepth 1 -maxdepth 1 -type d )
+for case in $cases
+do
+    casebase=$( basename "$case" )
+    tgt="${bundle}/components/${component}/testcases/${casebase}/expected-output"
+    if [ -d "$tgt" ]
+    then 
+        for output in $( ls "${case}/component" | grep -v ^_ )
+        do
+            cp -a $@ "${case}/component/${output}" "${tgt}/"
+        done
+    else
+        echo "$component $casebase does not have expected-output folder"
+    fi
+done

anduril/vm_report_status

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+cores=$( grep -c processor /proc/cpuinfo )
+
+# verbose mode
+tgtf=/mnt/csc-cloud/vm_state/${HOSTNAME}.long
+# debug
+if [ -z "$1" ]
+then touch "$tgtf"
+fi
+if [ -f "$tgtf" ]
+then echo "======== $HOSTNAME =======" > "$tgtf"
+     TZ="Europe/Helsinki" uptime >> "$tgtf"
+     echo "Cores: $cores" >> "$tgtf"
+     free -tg | head -n 2 >> "$tgtf"
+     ps a --format user| grep -v -e root -e USER | sort -u | tr '\n' ' ' >> "$tgtf"
+     echo "" >> "$tgtf"
+fi
+
+# concise mode
+
+tgtf=/mnt/csc-cloud/vm_state/${HOSTNAME}.short
+# debug
+if [ -z "$1" ]
+then touch "$tgtf"
+fi
+if [ -f "$tgtf" ]
+then echo -n "|$HOSTNAME L:" > "$tgtf"
+     load=$( cat /proc/loadavg | cut -d" " -f3 | tr  -d [:cntrl:] )  
+     echo -n "$load/$cores M:" >> "$tgtf"
+     # used + cached = total - free 
+     free=$(( $( grep ^MemFree /proc/meminfo | tr -d -c [:digit:] ) / 1048576  ))
+     cache=$(( $( grep ^Cached /proc/meminfo | tr -d -c [:digit:] ) / 1048576  ))
+     total=$(( $( grep ^MemTotal /proc/meminfo | tr -d -c [:digit:] ) / 1048576  ))
+     used=$(( $total -$free -$cache ))
+     echo -n "$used+$cache/$total U:" >> "$tgtf" 
+     ps a --format user| grep -v -e root -e USER | sort -u | wc -l | tr -c -d [:digit:] >> "$tgtf"
+     TZ="Europe/Helsinki" date "+ @%H:%M" >> "$tgtf" 
+# save history
+     echo -e "$HOSTNAME\t$load\t$used\t"$( date +%s ) >> /mnt/csc-cloud/vm_state/history/${HOSTNAME}
+fi
+
+
+