more metadata handlers in image_list

image_list.py

@@ -7,11 +7,13 @@ import sqlite3
 import subprocess
 import hashlib
 import traceback
+import csv
+import datetime
 from argparse import ArgumentParser 
 
 SQLFILE='list_of_images.sqlite'
 DESCFILE='descriptions.csv'
-IMGMATCH=re.compile('.*\.jpg$|.*\.jpeg$|.*\.png$|.*\.gif$',re.I)
+IMGMATCH=re.compile('.*\.jpg$|.*\.jpeg$|.*\.png$|.*\.gif$|.*\.tif$',re.I)
 BADDIRS=['_tn','_med']
 MINSIZE=0
 
@@ -29,8 +31,14 @@ def setup_options():
                       help="Print directory sizes. Argument is the path where directories are listed from.")
     parser.add_argument("--du-depth",type=str,action='store',dest="diskused_depth",default=1,
                       help="Depth of summarization for --du.")
+    parser.add_argument("--exportDesc",action="store",dest="export_descriptions",default=None,
+                      help="Walk through folders, and write "+DESCFILE+" in each folder. Format descriptions with {desc} {width}x{height} {red} {green} {blue} {Bred} {Bgreen} {Bblue} {size} {date} {name} {tags}")
     parser.add_argument("-f",action="store",dest="sqlfile",default=SQLFILE,
                       help="SQL file name to use [%(default)s]")
+    parser.add_argument("-i",action="store",dest="importfile",default=None,
+                      help="Import metadata from another sqlite database.")
+    parser.add_argument("--importDesc",action="store_true",dest="import_descriptions",default=False,
+                      help="Import description field from "+DESCFILE+" file in each folder")
     parser.add_argument("-l",action="store_true",dest="symlinks",default=False,
                      help="Follow symbolic links [%(default)s]")
     parser.add_argument("-m",type=int,dest="minsize",default=MINSIZE,
@@ -74,7 +82,8 @@ def setup_options():
        options.search or \
        options.diskused:
         options.add=not options.add
-    if options.tag:
+    if options.tag or\
+       options.importfile:
         options.add=False
     return options
 
@@ -99,10 +108,13 @@ def delete_nonexisting(sqlfile):
     db=conn.cursor()
     dbdel=conn.cursor()
     db.execute('SELECT file FROM list')
+    i=0
     for row in db:
         if not os.path.exists(row[0]):
             print('removing.. '+row[0])
-            dbdel.execute("DELETE FROM list where file == ?",(row[0],))            
+            dbdel.execute("DELETE FROM list where file == ?",(row[0],))
+            i+=1
+    print('Removed {0} entries'.format(i))
     conn.commit()
     return
 
@@ -112,13 +124,22 @@ def delete_data(sqlfile):
     db=conn.cursor()
     dbdel=conn.cursor()
     db.execute('''SELECT hash FROM data EXCEPT SELECT hash FROM list''')
-    for row in db:
+    i=0
+    for i,row in enumerate(db):
         dbdel.execute("DELETE FROM data where hash == ?",(row[0],))
     conn.commit()
+    print('Removed {0} metadata'.format(i))
+    
+    tagsbefore=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0]
     db.execute('''SELECT hash FROM tags EXCEPT SELECT hash FROM list''')
     for row in db:
         dbdel.execute("DELETE FROM tags where hash == ?",(row[0],))
+    db.execute('''DELETE FROM tags WHERE rowid NOT IN
+                    ( SELECT MIN(rowid) FROM tags GROUP BY hash,tag )''')
     conn.commit()
+    tagsafter=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0]
+    print('Removed {0} tags'.format(tagsbefore-tagsafter))
+    
     return
         
 def delete_files(files):
@@ -821,21 +842,117 @@ def humanize_size(size,precision=1):
         defPrecision=precision
     return "%.*f%s"%(defPrecision,size,suffixes[suffixIndex])
 
+def humanize_date(date):
+    if date==None:
+        return ''
+    return datetime.datetime.fromtimestamp(int(date)).strftime('%Y-%m-%d %H:%M:%S')
+
 def import_descriptions(options):
     """ Walk through the path from given [startpath] and read 
     any DESCFILE, importing the contents in the DB """
-    pass
+    conn=sqlite3.connect(options.sqlfile)
+    conn.text_factory=str
+    db=conn.cursor()
+    for path,dirs,files in os.walk(options.startpath,followlinks=options.symlinks):
+        dirs=clean_dirs(dirs)
+        if not options.symlinks:
+            files=clean_syms(files)
+        files.sort()
+        dirs.sort()
+        db_files=get_folder_contents(db,os.path.realpath(path)+'/')
+        if len(db_files)==0:
+            continue
+        if not os.path.exists( os.path.join(path,DESCFILE) ):
+            continue
+        read_file=open(os.path.join(path,DESCFILE),'r')
+        reader=csv.reader(read_file, dialect='excel-tab')
+        for row in reader:
+            if row[0] in db_files:
+                hash=file2hash(db,os.path.realpath(os.path.join(path,row[0])))
+                if hash==None:
+                    continue
+                db.execute("UPDATE data SET description=? \
+                            WHERE hash = ?",(row[1],hash))
+            conn.commit()
+        read_file.close()
 
 def export_descriptions(options):
-    """ Get unique paths from DB, matching [startpath], write
+    """ Walk through folders, and write DESCFILE csv descriptions.  """
-    DESCFILE for each file found. Export gets a format argument:
-     %wx%h %n %d """
      # width, height, basename, description
      #%R%G%B %S %F %D
      # Red Green Blue Sharpness Fingerprint Date(formatting?)
      # %s %H
      # filesize Hash
-    pass
+    conn=sqlite3.connect(options.sqlfile)
+    conn.text_factory=str
+    db=conn.cursor()
+    for path,dirs,files in os.walk(options.startpath,followlinks=options.symlinks):
+        dirs=clean_dirs(dirs)
+        if not options.symlinks:
+            files=clean_syms(files)
+        files.sort()
+        dirs.sort()
+        db_files=get_folder_contents(db,os.path.realpath(path)+'/')
+        if len(db_files)==0:
+            continue
+        print('Writing to '+os.path.join(path,DESCFILE))
+        # if exist DESCFILE
+        write_file=open(os.path.join(path,DESCFILE),'w')
+        writer=csv.writer(write_file, dialect='excel-tab')
+        writer.writerow(["File","Description"])
+        for f in db_files:
+            fullname=os.path.realpath(os.path.join(path,f))
+            hash=file2hash(db,fullname)
+            if hash==None:
+                continue
+            l=db.execute("SELECT * FROM list WHERE hash = ?",(hash,)).fetchall()[0]
+            d=db.execute("SELECT * FROM data WHERE hash = ?",(hash,)).fetchall()[0]
+            t=",".join([x[0] for x in db.execute("SELECT tag FROM tags WHERE hash = ?",(hash,)).fetchall()])
+            writer.writerow([f,description_parse(options.export_descriptions, l,d,t)])
+        write_file.close()
+
+def description_parse(s,l,d,t):
+    """{desc} {width}x{height} {red} {green} {blue} {Bred} {Bgreen} {Bblue} {size} {date} {name} {tags}"""
+    d=["" if x==None else x for x in d]
+    return s.format(
+        desc=d[1],
+        width=d[3],
+        height=d[4],
+        red=d[7],
+        green=d[8],
+        blue=d[9],
+        Bred=d[10],
+        Bgreen=d[11],
+        Bblue=d[12],
+        size=humanize_size(l[3]),
+        date=humanize_date(l[2]),
+        name=os.path.basename(l[0]),
+        tags=t,
+    )
+    
+
+def import_metadata(options):
+    """ import data table from another sqlite file"""
+    if not os.path.exists(options.importfile):
+        print("SQLite file {:} missing".format(options.importfile))
+        sys.exit(1)
+    
+    conn=sqlite3.connect(options.sqlfile)
+    conn.text_factory=str
+    db=conn.cursor()
+    before=db.execute("SELECT COUNT(hash) FROM data").fetchall()[0][0]
+    tagsbefore=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0]
+    db.execute("ATTACH ? as fromDB", (options.importfile, ))
+    db.execute("INSERT OR IGNORE INTO main.data SELECT * FROM fromDB.data")
+    db.execute("INSERT OR IGNORE INTO main.tags SELECT * FROM fromDB.tags")
+    conn.commit()
+    db.execute("""DELETE FROM main.tags WHERE rowid NOT IN
+                    ( SELECT MIN(rowid) FROM main.tags GROUP BY hash,tag )""")
+    conn.commit()
+    after=db.execute("SELECT COUNT(hash) FROM data").fetchall()[0][0]
+    tagsafter=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0]
+    
+    print("Imported {} metadata, {} tags.".format(after-before,tagsafter-tagsbefore))
 
 
 def main():
@@ -893,6 +1010,15 @@ def main():
             print_tag(options)
         else:
             add_tag(options)
+    if options.importfile:
+        print("Importing metadata")
+        import_metadata(options)
+    if options.import_descriptions:
+        print("Import descriptions")
+        import_descriptions(options)
+    if options.export_descriptions:
+        print("Export descriptions")
+        export_descriptions(options)
     sys.exit(0)
 
 if __name__ == "__main__":