192 lines
6.9 KiB
Python
Executable File
192 lines
6.9 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
import os
|
|
import sys
|
|
import re
|
|
import csv
|
|
import shutil
|
|
from argparse import ArgumentParser
|
|
|
|
filesearch=re.compile('^_index$')
|
|
STATEFILE='_state'
|
|
DRYSTATEFILE='_stateDryRun'
|
|
|
|
def setup_options():
|
|
usage='''%(prog)s [options]
|
|
Use rsync to copy your execution folder to a new place.
|
|
e.g. rsync -avP -e ssh user@server:/source/path/ target/path/
|
|
Then give a replacement pair -i and -o for each expected changed absolute path.
|
|
(multiple -i and -o are often required)
|
|
|
|
Each replace pair is tried, until a file with that replaced name is found.
|
|
'''
|
|
|
|
parser=ArgumentParser(usage=usage)
|
|
parser.add_argument("-d",type=str,dest="execpath",default=".",
|
|
help="Execution folder for anduril (location for _state), default: %(default)s")
|
|
parser.add_argument("-i",type=str,dest="inabs",action="append",default=["."],
|
|
help="Input absolute path prefix. e.g. /home1/user1/project/")
|
|
parser.add_argument("-o",type=str,dest="outabs",action="append",default=["."],
|
|
help="Output absolute path prefix. e.g. /home2/user2/different_project/")
|
|
parser.add_argument("-q",action="store_true",dest="query",default=False,
|
|
help="Compare "+STATEFILE+" and "+DRYSTATEFILE+" to "+
|
|
"see which components are not up-to-date. Use the anduril command: "+
|
|
"anduril run --dry ... to create the dry state file" )
|
|
return parser.parse_args()
|
|
|
|
def check_options(opts):
|
|
if not (os.path.isfile(os.path.join(opts.execpath,STATEFILE))):
|
|
print(STATEFILE+' file not found in folder '+opts.execpath)
|
|
sys.exit(1)
|
|
if opts.query:
|
|
if not (os.path.isfile(os.path.join(opts.execpath,DRYSTATEFILE))):
|
|
print(DRYSTATEFILE+' file not found in folder '+opts.execpath)
|
|
sys.exit(1)
|
|
if len(opts.inabs) is not len(opts.outabs):
|
|
print('A matching pair must be found for each -i/-o argument')
|
|
sys.exit(1)
|
|
if not opts.execpath.endswith('/'):
|
|
opts.execpath=opts.execpath+'/'
|
|
for i in xrange(len(opts.inabs)):
|
|
if not opts.inabs[i].endswith('/'):
|
|
opts.inabs[i]=opts.inabs[i]+'/'
|
|
for i in xrange(len(opts.outabs)):
|
|
if not opts.outabs[i].endswith('/'):
|
|
opts.outabs[i]=opts.outabs[i]+'/'
|
|
|
|
return opts
|
|
|
|
def getpathlist(path):
|
|
''' Returns a list of subfolders '''
|
|
list=os.listdir(path)
|
|
paths=[]
|
|
for d in list:
|
|
if (os.path.isdir(os.path.join(path,d))):
|
|
paths.append(d+'/')
|
|
return paths
|
|
|
|
def getfilelist(path):
|
|
''' Returns a list of files that might require change '''
|
|
list=os.listdir(path)
|
|
files=[]
|
|
for f in list:
|
|
if (filesearch.match(f)) and (os.path.isfile(os.path.join(path,f))):
|
|
files.append(f)
|
|
return files
|
|
|
|
def statefile(opts,path):
|
|
print('Parsing _state file')
|
|
shutil.copy2(os.path.join(path,STATEFILE), os.path.join(path,STATEFILE+'.bkp'))
|
|
statereader=csv.reader(open(os.path.join(path,STATEFILE),'rb'),
|
|
delimiter='\t',
|
|
doublequote=False,
|
|
escapechar='\\',
|
|
quoting=csv.QUOTE_NONE)
|
|
stateout=[]
|
|
for row in statereader:
|
|
rowout=row
|
|
if row[3].startswith('INPUT '):
|
|
newinput=row[3]
|
|
rowpstart=row[3].index(' P path=')
|
|
rowtstart=row[3].index(' TS in=')
|
|
rowpath=row[3][(rowpstart+7):rowtstart]
|
|
rowtime=row[3][(rowtstart+7):-1]
|
|
# time has a space at the end (and three zeros...)
|
|
print('INPUT found: "'+row[3]+'"')
|
|
found=False
|
|
for i in xrange(len(opts.inabs)):
|
|
newpath=rowpath.replace('='+opts.inabs[i],'='+opts.outabs[i])
|
|
if os.path.exists(newpath[1:]):
|
|
found=True
|
|
newtime=str(int(os.path.getmtime(newpath[1:])))+'000'
|
|
newinput=row[3].replace(rowpath,newpath,1).replace(rowtime,newtime,1)
|
|
print('NEW INPUT : "'+newinput+'"')
|
|
rowout[3]=newinput
|
|
break
|
|
if not found:
|
|
print('WARN: Could not find new INPUT, check your -i and -o arguments')
|
|
stateout.append(rowout)
|
|
statewriter=csv.writer(open(os.path.join(path,STATEFILE),'wb'),
|
|
delimiter='\t',
|
|
doublequote=False,
|
|
escapechar='\\',
|
|
quoting=csv.QUOTE_NONE)
|
|
statewriter.writerows(stateout)
|
|
|
|
return
|
|
|
|
def arrayreplace(path,filelist,opts):
|
|
header=['Key','File']
|
|
for f in filelist:
|
|
print('Modifying array: '+os.path.join(path,f))
|
|
arrayreader=csv.DictReader(open(os.path.join(path,f),'rb'),
|
|
delimiter='\t',
|
|
quotechar='"',
|
|
quoting=csv.QUOTE_ALL)
|
|
arrayout=[]
|
|
for row in arrayreader:
|
|
rowout=row
|
|
if os.path.exists(os.path.join(path,row['File'])):
|
|
arrayout.append(rowout)
|
|
continue
|
|
# File is a relative path, and exists - next iteration.
|
|
rowpath=row['File']
|
|
found=False
|
|
for i in xrange(len(opts.inabs)):
|
|
newpath=rowpath.replace(opts.inabs[i],opts.outabs[i],1)
|
|
if os.path.exists(newpath):
|
|
found=True
|
|
rowout['File']=newpath
|
|
break
|
|
if not found:
|
|
print('WARN: Could not find File '+rowpath+' in '+os.path.join(path,f)+', check your -i and -o arguments')
|
|
arrayout.append(rowout)
|
|
|
|
writer = csv.DictWriter(open(os.path.join(path,f),'wb'),
|
|
header,
|
|
delimiter='\t',
|
|
quotechar='"',
|
|
quoting=csv.QUOTE_MINIMAL)
|
|
writer.writerow(dict(zip(header,header)))
|
|
writer.writerows(arrayout)
|
|
|
|
return
|
|
|
|
def statequery(path):
|
|
statereader=csv.reader(open(os.path.join(path,DRYSTATEFILE),'rb'),
|
|
delimiter='\t',
|
|
doublequote=False,
|
|
escapechar='\\',
|
|
quoting=csv.QUOTE_NONE)
|
|
for row in statereader:
|
|
if row[1]=='NO':
|
|
print('Instance will run: '+row[0])
|
|
else:
|
|
print('Instance wont run: '+row[0])
|
|
|
|
return
|
|
|
|
|
|
|
|
def traverse(opts,path):
|
|
pathlist=getpathlist(path)
|
|
filelist=getfilelist(path)
|
|
arrayreplace(path,filelist,opts)
|
|
for p in pathlist:
|
|
traverse(opts,os.path.join(path,p))
|
|
return
|
|
|
|
def main():
|
|
opts=setup_options()
|
|
opts=check_options(opts)
|
|
|
|
if opts.query:
|
|
statequery(opts.execpath)
|
|
else:
|
|
traverse(opts,opts.execpath)
|
|
statefile(opts,opts.execpath)
|
|
return
|
|
|
|
|
|
main()
|