From 2aabfffbf30dd5a3f1024e2c2b3bfa291ef535a2 Mon Sep 17 00:00:00 2001 From: q Date: Mon, 9 Jun 2025 14:18:41 +0300 Subject: [PATCH] testing migration --- py-packages/imagelist2/imagelist2/__init__.py | 12 ++- py-packages/imagelist2/imagelist2/db.py | 73 ++++++++++++++++--- py-packages/imagelist2/imagelist2/image.py | 2 +- 3 files changed, 72 insertions(+), 15 deletions(-) diff --git a/py-packages/imagelist2/imagelist2/__init__.py b/py-packages/imagelist2/imagelist2/__init__.py index 03e107f..489c9c6 100644 --- a/py-packages/imagelist2/imagelist2/__init__.py +++ b/py-packages/imagelist2/imagelist2/__init__.py @@ -10,7 +10,7 @@ from imagelist2.db import DB, DBCachedWriter, sqlite_sqrt, sqlite_square from imagelist2.image import ImageBrokenError, ImageMeasure, is_image_extension from tqdm import tqdm -__version__ = "0.0.7" +__version__ = "0.0.8" SQLFILE = "image-list.sqlite" BADDIRS = ["_tn", "_med", ".tn", ".med"] MINSIZE = 0 @@ -166,7 +166,7 @@ class ImageList: to_delete = [] for row in self.db.cursor().execute( - """SELECT data.hash FROM data LEFT JOIN list ON list.hash = data.hash WHERE list.hash IS NULL""" + """SELECT data.hash FROM data LEFT JOIN list ON list.hash = data.hash WHERE list.hash IS NULL OR data.broken IS NULL""" ): to_delete.append(row[0]) @@ -218,7 +218,9 @@ class ImageList: OR data.R IS NULL ) AND - data.broken IS FALSE + ( + data.broken IS FALSE OR data.broken IS NULL + ) """ ) .fetchall() @@ -273,7 +275,8 @@ class ImageList: B = ?, BR = ?, BG = ?, - BB = ? + BB = ?, + broken = ? WHERE hash = ? """, ( @@ -285,6 +288,7 @@ class ImageList: image.colors["BR"], image.colors["BG"], image.colors["BB"], + image.broken, image.hash, ), ) diff --git a/py-packages/imagelist2/imagelist2/db.py b/py-packages/imagelist2/imagelist2/db.py index bc25963..4c6363e 100644 --- a/py-packages/imagelist2/imagelist2/db.py +++ b/py-packages/imagelist2/imagelist2/db.py @@ -17,19 +17,24 @@ class DB: def __init__(self, sqlfile): self.sqlfile = sqlfile self.root_path = os.path.dirname(os.path.realpath(sqlfile)) + self.migrated = False self.create_db() self.connect() def create_db(self): + from imagelist2 import __version__ + if os.path.exists(self.sqlfile): + self.migrate(__version__) return - conn = sqlite3.connect(self.sqlfile, timeout=30) - db = conn.cursor() - conn.text_factory = str - db.execute("CREATE TABLE list (file TEXT PRIMARY KEY,hash TEXT,date INTEGER,size INTEGER)") - db.execute( + db = sqlite3.connect(self.sqlfile, timeout=30) + db.text_factory = str + cursor = db.cursor() + + cursor.execute("CREATE TABLE list (file TEXT PRIMARY KEY,hash TEXT,date INTEGER,size INTEGER)") + cursor.execute( """CREATE TABLE data ( hash TEXT PRIMARY KEY, description TEXT, @@ -42,19 +47,66 @@ class DB: broken BOOLEAN )""" ) - db.execute("CREATE TABLE tags (hash TEXT,tag TEXT)") - db.execute( + cursor.execute("CREATE TABLE tags (hash TEXT,tag TEXT)") + cursor.execute( """CREATE VIEW files AS SELECT list.file, list.date, list.size, data.* FROM list LEFT JOIN data ON data.hash = list.hash""" ) - db.execute("CREATE UNIQUE INDEX data_hash ON data(hash)") - db.execute("CREATE UNIQUE INDEX list_file ON list(file)") + cursor.execute("CREATE TABLE config (key TEXT PRIMARY KEY, value TEXT)") + cursor.execute("CREATE UNIQUE INDEX data_hash ON data(hash)") + cursor.execute("CREATE UNIQUE INDEX list_file ON list(file)") + db.commit() + cursor = db.cursor() - conn.commit() + cursor.execute("INSERT INTO config (key,value) VALUES (?,?)", ("version", __version__)) + db.commit() return + def migrate(self, running_version): + """Before versions in config""" + try: + db = sqlite3.connect(self.sqlfile, timeout=30) + cursor = db.cursor() + config_version = cursor.execute("SELECT value FROM config WHERE key = 'version'").fetchall()[0][0] + if config_version == running_version: + # versions match + return + db.close() + except Exception: + # last version without config + config_version = "0.0.6" + self.migrated = True + if config_version == "0.0.6": # => 0.0.7 + try: + with sqlite3.connect(self.sqlfile, timeout=30) as db: + cursor = db.cursor() + cursor.execute("ALTER TABLE data ADD p_hash TEXT;") + db.commit() + except Exception: + pass + + with sqlite3.connect(self.sqlfile, timeout=30) as db: + cursor = db.cursor() + cursor.execute("ALTER TABLE data ADD broken BOOLEAN;") + cursor.execute("CREATE TABLE config (key TEXT PRIMARY KEY, value TEXT)") + config_version = "0.0.7" + cursor.execute("INSERT INTO config (key,value) VALUES (?,?)", ("version", config_version)) + cursor.execute("UPDATE data SET broken = ?;", (False,)) + db.commit() + + if config_version == "0.0.7": # => 0.0.8 + with sqlite3.connect(self.sqlfile, timeout=30) as db: + cursor = db.cursor() + config_version = "0.0.8" + cursor.execute("UPDATE config SET value = ? WHERE key = ?;", (config_version,"version")) + db.commit() + + print(f"Migrated to {config_version}. Restart", file=sys.stderr) + sys.exit(1) + + def connect(self): conn = sqlite3.connect(self.sqlfile, timeout=30) conn.text_factory = str @@ -183,6 +235,7 @@ class DBCachedWriter: cursor.execute(row["query"], row["values"]) self.db.conn.commit() except sqlite3.OperationalError as e: + print(e, file=sys.stderr) print("Writing failed, waiting for next writeout...", file=sys.stderr) self.cache_time = time.time() self.try_count += 1 diff --git a/py-packages/imagelist2/imagelist2/image.py b/py-packages/imagelist2/imagelist2/image.py index fdb74ab..2a7598a 100644 --- a/py-packages/imagelist2/imagelist2/image.py +++ b/py-packages/imagelist2/imagelist2/image.py @@ -144,7 +144,7 @@ class ImageMeasure: if len(self.image.shape) > 2: # BGR -> RGB self.image = np.flip(self.image, axis=2) - + self.broken = False if self.image_type == image_type: return self.image if image_type == "numpy":