diff --git a/image_list.py b/image_list.py index 733691e..8e85c96 100755 --- a/image_list.py +++ b/image_list.py @@ -7,11 +7,13 @@ import sqlite3 import subprocess import hashlib import traceback +import csv +import datetime from argparse import ArgumentParser SQLFILE='list_of_images.sqlite' DESCFILE='descriptions.csv' -IMGMATCH=re.compile('.*\.jpg$|.*\.jpeg$|.*\.png$|.*\.gif$',re.I) +IMGMATCH=re.compile('.*\.jpg$|.*\.jpeg$|.*\.png$|.*\.gif$|.*\.tif$',re.I) BADDIRS=['_tn','_med'] MINSIZE=0 @@ -29,8 +31,14 @@ def setup_options(): help="Print directory sizes. Argument is the path where directories are listed from.") parser.add_argument("--du-depth",type=str,action='store',dest="diskused_depth",default=1, help="Depth of summarization for --du.") + parser.add_argument("--exportDesc",action="store",dest="export_descriptions",default=None, + help="Walk through folders, and write "+DESCFILE+" in each folder. Format descriptions with {desc} {width}x{height} {red} {green} {blue} {Bred} {Bgreen} {Bblue} {size} {date} {name} {tags}") parser.add_argument("-f",action="store",dest="sqlfile",default=SQLFILE, help="SQL file name to use [%(default)s]") + parser.add_argument("-i",action="store",dest="importfile",default=None, + help="Import metadata from another sqlite database.") + parser.add_argument("--importDesc",action="store_true",dest="import_descriptions",default=False, + help="Import description field from "+DESCFILE+" file in each folder") parser.add_argument("-l",action="store_true",dest="symlinks",default=False, help="Follow symbolic links [%(default)s]") parser.add_argument("-m",type=int,dest="minsize",default=MINSIZE, @@ -74,7 +82,8 @@ def setup_options(): options.search or \ options.diskused: options.add=not options.add - if options.tag: + if options.tag or\ + options.importfile: options.add=False return options @@ -99,10 +108,13 @@ def delete_nonexisting(sqlfile): db=conn.cursor() dbdel=conn.cursor() db.execute('SELECT file FROM list') + i=0 for row in db: if not os.path.exists(row[0]): print('removing.. '+row[0]) - dbdel.execute("DELETE FROM list where file == ?",(row[0],)) + dbdel.execute("DELETE FROM list where file == ?",(row[0],)) + i+=1 + print('Removed {0} entries'.format(i)) conn.commit() return @@ -112,13 +124,22 @@ def delete_data(sqlfile): db=conn.cursor() dbdel=conn.cursor() db.execute('''SELECT hash FROM data EXCEPT SELECT hash FROM list''') - for row in db: + i=0 + for i,row in enumerate(db): dbdel.execute("DELETE FROM data where hash == ?",(row[0],)) conn.commit() + print('Removed {0} metadata'.format(i)) + + tagsbefore=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0] db.execute('''SELECT hash FROM tags EXCEPT SELECT hash FROM list''') for row in db: dbdel.execute("DELETE FROM tags where hash == ?",(row[0],)) + db.execute('''DELETE FROM tags WHERE rowid NOT IN + ( SELECT MIN(rowid) FROM tags GROUP BY hash,tag )''') conn.commit() + tagsafter=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0] + print('Removed {0} tags'.format(tagsbefore-tagsafter)) + return def delete_files(files): @@ -821,21 +842,117 @@ def humanize_size(size,precision=1): defPrecision=precision return "%.*f%s"%(defPrecision,size,suffixes[suffixIndex]) +def humanize_date(date): + if date==None: + return '' + return datetime.datetime.fromtimestamp(int(date)).strftime('%Y-%m-%d %H:%M:%S') + def import_descriptions(options): """ Walk through the path from given [startpath] and read any DESCFILE, importing the contents in the DB """ - pass + conn=sqlite3.connect(options.sqlfile) + conn.text_factory=str + db=conn.cursor() + for path,dirs,files in os.walk(options.startpath,followlinks=options.symlinks): + dirs=clean_dirs(dirs) + if not options.symlinks: + files=clean_syms(files) + files.sort() + dirs.sort() + db_files=get_folder_contents(db,os.path.realpath(path)+'/') + if len(db_files)==0: + continue + if not os.path.exists( os.path.join(path,DESCFILE) ): + continue + read_file=open(os.path.join(path,DESCFILE),'r') + reader=csv.reader(read_file, dialect='excel-tab') + for row in reader: + if row[0] in db_files: + hash=file2hash(db,os.path.realpath(os.path.join(path,row[0]))) + if hash==None: + continue + db.execute("UPDATE data SET description=? \ + WHERE hash = ?",(row[1],hash)) + conn.commit() + read_file.close() def export_descriptions(options): - """ Get unique paths from DB, matching [startpath], write - DESCFILE for each file found. Export gets a format argument: - %wx%h %n %d """ + """ Walk through folders, and write DESCFILE csv descriptions. """ # width, height, basename, description #%R%G%B %S %F %D # Red Green Blue Sharpness Fingerprint Date(formatting?) # %s %H # filesize Hash - pass + conn=sqlite3.connect(options.sqlfile) + conn.text_factory=str + db=conn.cursor() + for path,dirs,files in os.walk(options.startpath,followlinks=options.symlinks): + dirs=clean_dirs(dirs) + if not options.symlinks: + files=clean_syms(files) + files.sort() + dirs.sort() + db_files=get_folder_contents(db,os.path.realpath(path)+'/') + if len(db_files)==0: + continue + print('Writing to '+os.path.join(path,DESCFILE)) + # if exist DESCFILE + write_file=open(os.path.join(path,DESCFILE),'w') + writer=csv.writer(write_file, dialect='excel-tab') + writer.writerow(["File","Description"]) + for f in db_files: + fullname=os.path.realpath(os.path.join(path,f)) + hash=file2hash(db,fullname) + if hash==None: + continue + l=db.execute("SELECT * FROM list WHERE hash = ?",(hash,)).fetchall()[0] + d=db.execute("SELECT * FROM data WHERE hash = ?",(hash,)).fetchall()[0] + t=",".join([x[0] for x in db.execute("SELECT tag FROM tags WHERE hash = ?",(hash,)).fetchall()]) + writer.writerow([f,description_parse(options.export_descriptions, l,d,t)]) + write_file.close() + +def description_parse(s,l,d,t): + """{desc} {width}x{height} {red} {green} {blue} {Bred} {Bgreen} {Bblue} {size} {date} {name} {tags}""" + d=["" if x==None else x for x in d] + return s.format( + desc=d[1], + width=d[3], + height=d[4], + red=d[7], + green=d[8], + blue=d[9], + Bred=d[10], + Bgreen=d[11], + Bblue=d[12], + size=humanize_size(l[3]), + date=humanize_date(l[2]), + name=os.path.basename(l[0]), + tags=t, + ) + + +def import_metadata(options): + """ import data table from another sqlite file""" + if not os.path.exists(options.importfile): + print("SQLite file {:} missing".format(options.importfile)) + sys.exit(1) + + conn=sqlite3.connect(options.sqlfile) + conn.text_factory=str + db=conn.cursor() + before=db.execute("SELECT COUNT(hash) FROM data").fetchall()[0][0] + tagsbefore=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0] + db.execute("ATTACH ? as fromDB", (options.importfile, )) + db.execute("INSERT OR IGNORE INTO main.data SELECT * FROM fromDB.data") + db.execute("INSERT OR IGNORE INTO main.tags SELECT * FROM fromDB.tags") + conn.commit() + db.execute("""DELETE FROM main.tags WHERE rowid NOT IN + ( SELECT MIN(rowid) FROM main.tags GROUP BY hash,tag )""") + conn.commit() + after=db.execute("SELECT COUNT(hash) FROM data").fetchall()[0][0] + tagsafter=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0] + + print("Imported {} metadata, {} tags.".format(after-before,tagsafter-tagsbefore)) def main(): @@ -893,6 +1010,15 @@ def main(): print_tag(options) else: add_tag(options) + if options.importfile: + print("Importing metadata") + import_metadata(options) + if options.import_descriptions: + print("Import descriptions") + import_descriptions(options) + if options.export_descriptions: + print("Export descriptions") + export_descriptions(options) sys.exit(0) if __name__ == "__main__":