testing migration

py-packages/imagelist2/imagelist2/__init__.py

@@ -10,7 +10,7 @@ from imagelist2.db import DB, DBCachedWriter, sqlite_sqrt, sqlite_square
 from imagelist2.image import ImageBrokenError, ImageMeasure, is_image_extension
 from tqdm import tqdm
 
-__version__ = "0.0.7"
+__version__ = "0.0.8"
 SQLFILE = "image-list.sqlite"
 BADDIRS = ["_tn", "_med", ".tn", ".med"]
 MINSIZE = 0
@@ -166,7 +166,7 @@ class ImageList:
 
         to_delete = []
         for row in self.db.cursor().execute(
-            """SELECT data.hash FROM data LEFT JOIN list ON list.hash = data.hash WHERE list.hash IS NULL"""
+            """SELECT data.hash FROM data LEFT JOIN list ON list.hash = data.hash WHERE list.hash IS NULL OR data.broken IS NULL"""
         ):
             to_delete.append(row[0])
 
@@ -218,7 +218,9 @@ class ImageList:
                     OR data.R IS NULL
                 )
                 AND
-                data.broken IS FALSE
+                (
+                    data.broken IS FALSE OR data.broken IS NULL
+                )
         """
             )
             .fetchall()
@@ -273,7 +275,8 @@ class ImageList:
                     B = ?,
                     BR = ?,
                     BG = ?,
-                    BB = ?
+                    BB = ?,
+                    broken = ?
                  WHERE hash = ?
               """,
                 (
@@ -285,6 +288,7 @@ class ImageList:
                     image.colors["BR"],
                     image.colors["BG"],
                     image.colors["BB"],
+                    image.broken,
                     image.hash,
                 ),
             )

py-packages/imagelist2/imagelist2/db.py

@@ -17,19 +17,24 @@ class DB:
     def __init__(self, sqlfile):
         self.sqlfile = sqlfile
         self.root_path = os.path.dirname(os.path.realpath(sqlfile))
+        self.migrated = False
         self.create_db()
         self.connect()
 
     def create_db(self):
 
+        from imagelist2 import __version__
+
         if os.path.exists(self.sqlfile):
+            self.migrate(__version__)
             return
 
-        conn = sqlite3.connect(self.sqlfile, timeout=30)
+        db = sqlite3.connect(self.sqlfile, timeout=30)
-        db = conn.cursor()
+        db.text_factory = str
-        conn.text_factory = str
+        cursor = db.cursor()
-        db.execute("CREATE TABLE list (file TEXT PRIMARY KEY,hash TEXT,date INTEGER,size INTEGER)")
+
-        db.execute(
+        cursor.execute("CREATE TABLE list (file TEXT PRIMARY KEY,hash TEXT,date INTEGER,size INTEGER)")
+        cursor.execute(
             """CREATE TABLE data (
                 hash TEXT PRIMARY KEY,
                 description TEXT,
@@ -42,19 +47,66 @@ class DB:
                 broken BOOLEAN
             )"""
         )
-        db.execute("CREATE TABLE tags (hash TEXT,tag TEXT)")
+        cursor.execute("CREATE TABLE tags (hash TEXT,tag TEXT)")
-        db.execute(
+        cursor.execute(
             """CREATE VIEW files AS
               SELECT list.file, list.date, list.size, data.*
               FROM list
               LEFT JOIN data ON data.hash = list.hash"""
         )
-        db.execute("CREATE UNIQUE INDEX data_hash ON data(hash)")
+        cursor.execute("CREATE TABLE config (key TEXT PRIMARY KEY, value TEXT)")
-        db.execute("CREATE UNIQUE INDEX list_file ON list(file)")
+        cursor.execute("CREATE UNIQUE INDEX data_hash ON data(hash)")
+        cursor.execute("CREATE UNIQUE INDEX list_file ON list(file)")
+        db.commit()
+        cursor = db.cursor()
 
-        conn.commit()
+        cursor.execute("INSERT INTO config (key,value) VALUES (?,?)", ("version", __version__))
+        db.commit()
         return
 
+    def migrate(self, running_version):
+        """Before versions in config"""
+        try:
+            db = sqlite3.connect(self.sqlfile, timeout=30)
+            cursor = db.cursor()
+            config_version = cursor.execute("SELECT value FROM config WHERE key = 'version'").fetchall()[0][0]
+            if config_version == running_version:
+                # versions match
+                return
+            db.close()
+        except Exception:
+            # last version without config
+            config_version = "0.0.6"
+        self.migrated = True
+        if config_version == "0.0.6": # => 0.0.7
+            try:
+                with sqlite3.connect(self.sqlfile, timeout=30) as db:
+                    cursor = db.cursor()
+                    cursor.execute("ALTER TABLE data ADD p_hash TEXT;")
+                    db.commit()
+            except Exception:
+                pass
+
+            with sqlite3.connect(self.sqlfile, timeout=30) as db:
+                cursor = db.cursor()
+                cursor.execute("ALTER TABLE data ADD broken BOOLEAN;")
+                cursor.execute("CREATE TABLE config (key TEXT PRIMARY KEY, value TEXT)")
+                config_version = "0.0.7"
+                cursor.execute("INSERT INTO config (key,value) VALUES (?,?)", ("version", config_version))
+                cursor.execute("UPDATE data SET broken = ?;", (False,))
+                db.commit()
+
+        if config_version == "0.0.7": # => 0.0.8
+            with sqlite3.connect(self.sqlfile, timeout=30) as db:
+                cursor = db.cursor()
+                config_version = "0.0.8"
+                cursor.execute("UPDATE config SET value = ? WHERE key = ?;", (config_version,"version"))
+                db.commit()
+
+        print(f"Migrated to {config_version}. Restart", file=sys.stderr)
+        sys.exit(1)
+
+
     def connect(self):
         conn = sqlite3.connect(self.sqlfile, timeout=30)
         conn.text_factory = str
@@ -183,6 +235,7 @@ class DBCachedWriter:
                     cursor.execute(row["query"], row["values"])
                 self.db.conn.commit()
             except sqlite3.OperationalError as e:
+                print(e, file=sys.stderr)
                 print("Writing failed, waiting for next writeout...", file=sys.stderr)
                 self.cache_time = time.time()
                 self.try_count += 1

py-packages/imagelist2/imagelist2/image.py

@@ -144,7 +144,7 @@ class ImageMeasure:
                 if len(self.image.shape) > 2:
                     # BGR -> RGB
                     self.image = np.flip(self.image, axis=2)
-
+        self.broken = False
         if self.image_type == image_type:
             return self.image
         if image_type == "numpy":