moving projects

anduril/exec_copy.py

@@ -0,0 +1,191 @@
+#!/usr/bin/python
+
+import os
+import sys
+import re
+import csv
+import shutil
+from argparse import ArgumentParser
+
+filesearch=re.compile('^_index$')
+STATEFILE='_state'
+DRYSTATEFILE='_stateDryRun'
+
+def setup_options():
+    usage='''%(prog)s [options]
+    Use rsync to copy your execution folder to a new place. 
+    e.g.  rsync -avP -e ssh user@server:/source/path/ target/path/
+    Then give a replacement pair -i and -o for each expected changed absolute path.
+    (multiple -i and -o are often required)
+    
+    Each replace pair is tried, until a file with that replaced name is found.
+    '''
+
+    parser=ArgumentParser(usage=usage)
+    parser.add_argument("-d",type=str,dest="execpath",default=".",
+                     help="Execution folder for anduril (location for _state), default: %(default)s")
+    parser.add_argument("-i",type=str,dest="inabs",action="append",default=["."],
+                     help="Input absolute path prefix. e.g. /home1/user1/project/")
+    parser.add_argument("-o",type=str,dest="outabs",action="append",default=["."],
+                     help="Output absolute path prefix. e.g. /home2/user2/different_project/")
+    parser.add_argument("-q",action="store_true",dest="query",default=False,
+                     help="Compare "+STATEFILE+" and "+DRYSTATEFILE+" to "+
+                          "see which components are not up-to-date. Use the anduril command: "+
+                          "anduril run --dry ... to create the dry state file" )
+    return parser.parse_args()
+    
+def check_options(opts):
+    if not (os.path.isfile(os.path.join(opts.execpath,STATEFILE))):
+        print(STATEFILE+' file not found in folder '+opts.execpath)
+        sys.exit(1)
+    if opts.query:
+        if not (os.path.isfile(os.path.join(opts.execpath,DRYSTATEFILE))):
+            print(DRYSTATEFILE+' file not found in folder '+opts.execpath)
+            sys.exit(1)        
+    if len(opts.inabs) is not len(opts.outabs):
+        print('A matching pair must be found for each -i/-o argument')
+        sys.exit(1)
+    if not opts.execpath.endswith('/'):
+         opts.execpath=opts.execpath+'/'
+    for i in xrange(len(opts.inabs)):
+        if not opts.inabs[i].endswith('/'):
+            opts.inabs[i]=opts.inabs[i]+'/'
+    for i in xrange(len(opts.outabs)):
+        if not opts.outabs[i].endswith('/'):
+            opts.outabs[i]=opts.outabs[i]+'/'
+           
+    return opts
+    
+def getpathlist(path):
+    ''' Returns a list of subfolders '''
+    list=os.listdir(path)
+    paths=[]
+    for d in list:
+        if (os.path.isdir(os.path.join(path,d))):
+            paths.append(d+'/')
+    return paths  
+
+def getfilelist(path):
+    ''' Returns a list of files that might require change '''
+    list=os.listdir(path)
+    files=[]
+    for f in list:
+        if (filesearch.match(f)) and (os.path.isfile(os.path.join(path,f))):
+            files.append(f)
+    return files    
+
+def statefile(opts,path):
+    print('Parsing _state file')
+    shutil.copy2(os.path.join(path,STATEFILE), os.path.join(path,STATEFILE+'.bkp'))
+    statereader=csv.reader(open(os.path.join(path,STATEFILE),'rb'),
+                        delimiter='\t',
+                        doublequote=False,
+                        escapechar='\\',
+                        quoting=csv.QUOTE_NONE)
+    stateout=[]
+    for row in statereader:
+        rowout=row
+        if row[3].startswith('INPUT '):
+            newinput=row[3]
+            rowpstart=row[3].index(' P path=')
+            rowtstart=row[3].index(' TS in=')
+            rowpath=row[3][(rowpstart+7):rowtstart]
+            rowtime=row[3][(rowtstart+7):-1]
+            # time has a space at the end (and three zeros...)
+            print('INPUT found: "'+row[3]+'"')
+            found=False
+            for i in xrange(len(opts.inabs)):
+                newpath=rowpath.replace('='+opts.inabs[i],'='+opts.outabs[i])
+                if os.path.exists(newpath[1:]):
+                    found=True
+                    newtime=str(int(os.path.getmtime(newpath[1:])))+'000'
+                    newinput=row[3].replace(rowpath,newpath,1).replace(rowtime,newtime,1)
+                    print('NEW INPUT  : "'+newinput+'"')
+                    rowout[3]=newinput
+                    break
+            if not found:
+                print('WARN: Could not find new INPUT, check your -i and -o arguments')
+        stateout.append(rowout)    
+    statewriter=csv.writer(open(os.path.join(path,STATEFILE),'wb'),
+                        delimiter='\t',
+                        doublequote=False,
+                        escapechar='\\',
+                        quoting=csv.QUOTE_NONE)
+    statewriter.writerows(stateout)
+        
+    return
+
+def arrayreplace(path,filelist,opts):
+    header=['Key','File']
+    for f in filelist:
+        print('Modifying array: '+os.path.join(path,f))
+        arrayreader=csv.DictReader(open(os.path.join(path,f),'rb'),
+                               delimiter='\t',
+                               quotechar='"', 
+                               quoting=csv.QUOTE_ALL)
+        arrayout=[]
+        for row in arrayreader:
+            rowout=row
+            if os.path.exists(os.path.join(path,row['File'])):
+                arrayout.append(rowout)
+                continue
+                # File is a relative path, and exists - next iteration.
+            rowpath=row['File']
+            found=False
+            for i in xrange(len(opts.inabs)):
+                newpath=rowpath.replace(opts.inabs[i],opts.outabs[i],1)
+                if os.path.exists(newpath):
+                    found=True
+                    rowout['File']=newpath
+                    break
+            if not found:
+                print('WARN: Could not find File '+rowpath+' in '+os.path.join(path,f)+', check your -i and -o arguments')
+            arrayout.append(rowout)
+        
+        writer = csv.DictWriter(open(os.path.join(path,f),'wb'),
+                            header,
+                            delimiter='\t',
+                            quotechar='"',
+                            quoting=csv.QUOTE_MINIMAL)
+        writer.writerow(dict(zip(header,header)))
+        writer.writerows(arrayout)
+        
+    return
+    
+def statequery(path):
+    statereader=csv.reader(open(os.path.join(path,DRYSTATEFILE),'rb'),
+                        delimiter='\t',
+                        doublequote=False,
+                        escapechar='\\',
+                        quoting=csv.QUOTE_NONE)
+    for row in statereader:
+        if row[1]=='NO':
+            print('Instance will run: '+row[0])
+        else:
+            print('Instance wont run: '+row[0])
+   
+    return
+    
+        
+    
+def traverse(opts,path):
+    pathlist=getpathlist(path)
+    filelist=getfilelist(path)
+    arrayreplace(path,filelist,opts)
+    for p in pathlist:
+        traverse(opts,os.path.join(path,p))
+    return
+    
+def main():
+    opts=setup_options()
+    opts=check_options(opts)
+
+    if opts.query:
+        statequery(opts.execpath)
+    else:
+        traverse(opts,opts.execpath)
+        statefile(opts,opts.execpath)
+    return
+
+
+main()