moonq/q-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

more metadata handlers in image_list

Browse Source
This commit is contained in:
ville rantanen
2015-02-13 12:03:57 +02:00
parent cc495f1847
commit f2cb6fbd08
1 changed files with 135 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
image_list.py
View File

@@ -7,11 +7,13 @@ import sqlite3
import subprocess
import hashlib
import traceback
import csv
import datetime
from argparse import ArgumentParser
SQLFILE='list_of_images.sqlite'
DESCFILE='descriptions.csv'
IMGMATCH=re.compile('.*\.jpg$|.*\.jpeg$|.*\.png$|.*\.gif$',re.I)
IMGMATCH=re.compile('.*\.jpg$|.*\.jpeg$|.*\.png$|.*\.gif$|.*\.tif$',re.I)
BADDIRS=['_tn','_med']
MINSIZE=0
@@ -29,8 +31,14 @@ def setup_options():
help="Print directory sizes. Argument is the path where directories are listed from.")
parser.add_argument("--du-depth",type=str,action='store',dest="diskused_depth",default=1,
help="Depth of summarization for --du.")
parser.add_argument("--exportDesc",action="store",dest="export_descriptions",default=None,
help="Walk through folders, and write "+DESCFILE+" in each folder. Format descriptions with {desc} {width}x{height} {red} {green} {blue} {Bred} {Bgreen} {Bblue} {size} {date} {name} {tags}")
parser.add_argument("-f",action="store",dest="sqlfile",default=SQLFILE,
help="SQL file name to use [%(default)s]")
parser.add_argument("-i",action="store",dest="importfile",default=None,
help="Import metadata from another sqlite database.")
parser.add_argument("--importDesc",action="store_true",dest="import_descriptions",default=False,
help="Import description field from "+DESCFILE+" file in each folder")
parser.add_argument("-l",action="store_true",dest="symlinks",default=False,
help="Follow symbolic links [%(default)s]")
parser.add_argument("-m",type=int,dest="minsize",default=MINSIZE,
@@ -74,7 +82,8 @@ def setup_options():
options.search or \
options.diskused:
options.add=not options.add
if options.tag:
if options.tag or\
options.importfile:
options.add=False
return options
@@ -99,10 +108,13 @@ def delete_nonexisting(sqlfile):
db=conn.cursor()
dbdel=conn.cursor()
db.execute('SELECT file FROM list')
i=0
for row in db:
if not os.path.exists(row[0]):
print('removing.. '+row[0])
dbdel.execute("DELETE FROM list where file == ?",(row[0],))
i+=1
print('Removed {0} entries'.format(i))
conn.commit()
return
@@ -112,13 +124,22 @@ def delete_data(sqlfile):
db=conn.cursor()
dbdel=conn.cursor()
db.execute('''SELECT hash FROM data EXCEPT SELECT hash FROM list''')
for row in db:
i=0
for i,row in enumerate(db):
dbdel.execute("DELETE FROM data where hash == ?",(row[0],))
conn.commit()
print('Removed {0} metadata'.format(i))
tagsbefore=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0]
db.execute('''SELECT hash FROM tags EXCEPT SELECT hash FROM list''')
for row in db:
dbdel.execute("DELETE FROM tags where hash == ?",(row[0],))
db.execute('''DELETE FROM tags WHERE rowid NOT IN
( SELECT MIN(rowid) FROM tags GROUP BY hash,tag )''')
conn.commit()
tagsafter=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0]
print('Removed {0} tags'.format(tagsbefore-tagsafter))
return
def delete_files(files):
@@ -821,21 +842,117 @@ def humanize_size(size,precision=1):
defPrecision=precision
return "%.*f%s"%(defPrecision,size,suffixes[suffixIndex])
def humanize_date(date):
if date==None:
return ''
return datetime.datetime.fromtimestamp(int(date)).strftime('%Y-%m-%d %H:%M:%S')
def import_descriptions(options):
""" Walk through the path from given [startpath] and read
any DESCFILE, importing the contents in the DB """
pass
conn=sqlite3.connect(options.sqlfile)
conn.text_factory=str
db=conn.cursor()
for path,dirs,files in os.walk(options.startpath,followlinks=options.symlinks):
dirs=clean_dirs(dirs)
if not options.symlinks:
files=clean_syms(files)
files.sort()
dirs.sort()
db_files=get_folder_contents(db,os.path.realpath(path)+'/')
if len(db_files)==0:
continue
if not os.path.exists( os.path.join(path,DESCFILE) ):
continue
read_file=open(os.path.join(path,DESCFILE),'r')
reader=csv.reader(read_file, dialect='excel-tab')
for row in reader:
if row[0] in db_files:
hash=file2hash(db,os.path.realpath(os.path.join(path,row[0])))
if hash==None:
continue
db.execute("UPDATE data SET description=? \
WHERE hash = ?",(row[1],hash))
conn.commit()
read_file.close()
def export_descriptions(options):
""" Get unique paths from DB, matching [startpath], write
DESCFILE for each file found. Export gets a format argument:
%wx%h %n %d """
""" Walk through folders, and write DESCFILE csv descriptions. """
# width, height, basename, description
#%R%G%B %S %F %D
# Red Green Blue Sharpness Fingerprint Date(formatting?)
# %s %H
# filesize Hash
pass
conn=sqlite3.connect(options.sqlfile)
conn.text_factory=str
db=conn.cursor()
for path,dirs,files in os.walk(options.startpath,followlinks=options.symlinks):
dirs=clean_dirs(dirs)
if not options.symlinks:
files=clean_syms(files)
files.sort()
dirs.sort()
db_files=get_folder_contents(db,os.path.realpath(path)+'/')
if len(db_files)==0:
continue
print('Writing to '+os.path.join(path,DESCFILE))
# if exist DESCFILE
write_file=open(os.path.join(path,DESCFILE),'w')
writer=csv.writer(write_file, dialect='excel-tab')
writer.writerow(["File","Description"])
for f in db_files:
fullname=os.path.realpath(os.path.join(path,f))
hash=file2hash(db,fullname)
if hash==None:
continue
l=db.execute("SELECT * FROM list WHERE hash = ?",(hash,)).fetchall()[0]
d=db.execute("SELECT * FROM data WHERE hash = ?",(hash,)).fetchall()[0]
t=",".join([x[0] for x in db.execute("SELECT tag FROM tags WHERE hash = ?",(hash,)).fetchall()])
writer.writerow([f,description_parse(options.export_descriptions, l,d,t)])
write_file.close()
def description_parse(s,l,d,t):
"""{desc} {width}x{height} {red} {green} {blue} {Bred} {Bgreen} {Bblue} {size} {date} {name} {tags}"""
d=["" if x==None else x for x in d]
return s.format(
desc=d[1],
width=d[3],
height=d[4],
red=d[7],
green=d[8],
blue=d[9],
Bred=d[10],
Bgreen=d[11],
Bblue=d[12],
size=humanize_size(l[3]),
date=humanize_date(l[2]),
name=os.path.basename(l[0]),
tags=t,
)
def import_metadata(options):
""" import data table from another sqlite file"""
if not os.path.exists(options.importfile):
print("SQLite file {:} missing".format(options.importfile))
sys.exit(1)
conn=sqlite3.connect(options.sqlfile)
conn.text_factory=str
db=conn.cursor()
before=db.execute("SELECT COUNT(hash) FROM data").fetchall()[0][0]
tagsbefore=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0]
db.execute("ATTACH ? as fromDB", (options.importfile, ))
db.execute("INSERT OR IGNORE INTO main.data SELECT * FROM fromDB.data")
db.execute("INSERT OR IGNORE INTO main.tags SELECT * FROM fromDB.tags")
conn.commit()
db.execute("""DELETE FROM main.tags WHERE rowid NOT IN
( SELECT MIN(rowid) FROM main.tags GROUP BY hash,tag )""")
conn.commit()
after=db.execute("SELECT COUNT(hash) FROM data").fetchall()[0][0]
tagsafter=db.execute("SELECT COUNT(hash) FROM tags").fetchall()[0][0]
print("Imported {} metadata, {} tags.".format(after-before,tagsafter-tagsbefore))
def main():
@@ -893,6 +1010,15 @@ def main():
print_tag(options)
else:
add_tag(options)
if options.importfile:
print("Importing metadata")
import_metadata(options)
if options.import_descriptions:
print("Import descriptions")
import_descriptions(options)
if options.export_descriptions:
print("Export descriptions")
export_descriptions(options)
sys.exit(0)
if __name__ == "__main__":