Introduce new migration procedure.

This creates a new migration module that include upgrade logic files, one file for each DB version. It should be noted that this will not support downgrades (the previous method didn't really support them either)

config.py

@@ -19,6 +19,10 @@
 expansionVersion = "1.0"
 evemonMinVersion = "4081"
 
+# Database version (int ONLY)
+# Increment every time we need to flag for user database upgrade/modification
+dbversion = 1
+
 pyfaPath = None
 savePath = None
 staticPath = None

eos/db/__init__.py

@@ -49,7 +49,6 @@ class ReadOnlyException(Exception):
 
     saveddata_meta = MetaData()
     saveddata_meta.bind = saveddata_engine
-    migration.update(saveddata_engine)
     saveddata_session = sessionmaker(bind=saveddata_engine, autoflush=False, expire_on_commit=False)()
 
 # Lock controlling any changes introduced to session

eos/db/migration.py

@@ -1,76 +1,21 @@
-import sqlalchemy
+import config
 
-def update(saveddata_engine):
-    checkPriceFailures(saveddata_engine)
-    checkApiDefaultChar(saveddata_engine)
-    checkFitBooster(saveddata_engine)
-    checktargetResists(saveddata_engine)
-
-def checkPriceFailures(saveddata_engine):
-    # Check if we have 'failed' column
-    try:
-        saveddata_engine.execute("SELECT failed FROM prices")
-    except sqlalchemy.exc.DatabaseError:
-        # As we don't have any important data there, let's just drop
-        # and recreate whole table
-        from eos.db.saveddata.price import prices_table
-        # Attempt to drop/create table only if it's already there
-        try:
-            prices_table.drop(saveddata_engine)
-            prices_table.create(saveddata_engine)
-        except sqlalchemy.exc.DatabaseError:
-            pass
+def getVersion(db):
+    cursor = db.execute('PRAGMA user_version')
+    return cursor.fetchone()[0]
 
+def update(saveddata_engine):
+    currversion = getVersion(saveddata_engine)
 
-def checkApiDefaultChar(saveddata_engine):
-    try:
-        saveddata_engine.execute("SELECT * FROM characters LIMIT 1")
-    # If table doesn't exist, it means we're doing everything from scratch
-    # and sqlalchemy will process everything as needed
-    except sqlalchemy.exc.DatabaseError:
-        pass
-    # If not, we're running on top of existing DB
-    else:
-        # Check that we have columns
-        try:
-            saveddata_engine.execute("SELECT defaultChar, chars FROM characters LIMIT 1")
-        # If we don't, create them
-        # This is ugly as hell, but we can't use proper migrate packages as it
-        # will require us to rebuild skeletons, including mac
-        except sqlalchemy.exc.DatabaseError:
-            saveddata_engine.execute("ALTER TABLE characters ADD COLUMN defaultChar INTEGER;")
-            saveddata_engine.execute("ALTER TABLE characters ADD COLUMN chars VARCHAR;")
+    if currversion == config.dbversion:
+        return
 
-def checkFitBooster(saveddata_engine):
-    try:
-        saveddata_engine.execute("SELECT * FROM fits LIMIT 1")
-    # If table doesn't exist, it means we're doing everything from scratch
-    # and sqlalchemy will process everything as needed
-    except sqlalchemy.exc.DatabaseError:
-        pass
-    # If not, we're running on top of existing DB
-    else:
-        # Check that we have columns
-        try:
-            saveddata_engine.execute("SELECT booster FROM fits LIMIT 1")
-        # If we don't, create them
-        except sqlalchemy.exc.DatabaseError:
-            saveddata_engine.execute("ALTER TABLE fits ADD COLUMN booster BOOLEAN;")
-        # Set NULL data to 0 (needed in case of downgrade, see GH issue #62
-        saveddata_engine.execute("UPDATE fits SET booster = 0 WHERE booster IS NULL;")
+    if currversion < config.dbversion:
+        for version in xrange(currversion, config.dbversion):
+            module = __import__('eos.db.migrations.upgrade%d'%(version+1), fromlist=True)
+            upgrade = getattr(module, "upgrade", False)
+            if upgrade:
+                upgrade(saveddata_engine)
 
-def checktargetResists(saveddata_engine):
-    try:
-        saveddata_engine.execute("SELECT * FROM fits LIMIT 1")
-    # If table doesn't exist, it means we're doing everything from scratch
-    # and sqlalchemy will process everything as needed
-    except sqlalchemy.exc.DatabaseError:
-        pass
-    # If not, we're running on top of existing DB
-    else:
-        # Check that we have columns
-        try:
-            saveddata_engine.execute("SELECT targetResistsID FROM fits LIMIT 1")
-        # If we don't, create them
-        except sqlalchemy.exc.DatabaseError:
-            saveddata_engine.execute("ALTER TABLE fits ADD COLUMN targetResistsID INTEGER;")
+        # when all is said and done, set version to current
+        saveddata_engine.execute('PRAGMA user_version = %d'%config.dbversion)

eos/db/migrations/__init__.py

@@ -0,0 +1,9 @@
+"""
+The migration module includes migration logic to update database scheme and/or
+data for the user database.
+
+To create a migration, simply create a file upgrade<migration number>.py and
+define an upgrade() function with the logic. Please note that there must be as
+many upgrade files as there are database versions (version 5 would include
+upgrade files 1-5)
+"""

eos/db/migrations/upgrade1.py

@@ -0,0 +1,92 @@
+"""
+Migration 1
+
+- Alters fits table to introduce target resist attribute
+- Converts modules based on Oceanus Module Tiericide
+    Some modules have been deleted, which causes pyfa to crash when fits are
+    loaded as they no longer exist in the database. We therefore replace these
+    modules with their new replacements
+
+    Based on http://community.eveonline.com/news/patch-notes/patch-notes-for-oceanus/
+    and output of itemDiff.py
+"""
+
+CONVERSIONS = {
+    6135: [  # Scoped Cargo Scanner
+        6133,  # Interior Type-E Cargo Identifier
+    ],
+    6527: [  # Compact Ship Scanner
+        6525,  # Ta3 Perfunctory Vessel Probe
+        6529,  # Speculative Ship Identifier I
+        6531,  # Practical Type-E Ship Probe
+    ],
+    6569: [  # Scoped Survey Scanner
+        6567,  # ML-3 Amphilotite Mining Probe
+        6571,  # Rock-Scanning Sensor Array I
+        6573,  # 'Dactyl' Type-E Asteroid Analyzer
+    ],
+    509: [  # 'Basic' Capacitor Flux Coil
+        8163,  # Partial Power Plant Manager: Capacitor Flux
+        8165,  # Alpha Reactor Control: Capacitor Flux
+        8167,  # Type-E Power Core Modification: Capacitor Flux
+        8169,  # Marked Generator Refitting: Capacitor Flux
+    ],
+    8135: [  # Restrained Capacitor Flux Coil
+        8131,  # Local Power Plant Manager: Capacitor Flux I
+    ],
+    8133: [  # Compact Capacitor Flux Coil
+        8137,  # Mark I Generator Refitting: Capacitor Flux
+    ],
+    3469: [  # Basic Co-Processor
+        8744,  # Nanoelectrical Co-Processor
+        8743,  # Nanomechanical CPU Enhancer
+        8746,  # Quantum Co-Processor
+        8745,  # Photonic CPU Enhancer
+        15425,  # Naiyon's Modified Co-Processor (never existed but convert
+                # anyway as some fits may include it)
+    ],
+    8748: [  # Upgraded Co-Processor
+        8747,  # Nanomechanical CPU Enhancer I
+        8750,  # Quantum Co-Processor I
+        8749,  # Photonic CPU Enhancer I
+    ],
+    1351: [  # Basic Reactor Control Unit
+        8251,  # Partial Power Plant Manager: Reaction Control
+        8253,  # Alpha Reactor Control: Reaction Control
+        8257,  # Marked Generator Refitting: Reaction Control
+    ],
+    8263: [  # Compact Reactor Control Unit
+        8259,  # Local Power Plant Manager: Reaction Control I
+        8265,  # Mark I Generator Refitting: Reaction Control
+        8261,  # Beta Reactor Control: Reaction Control I
+    ],
+    16537: [  # Compact Micro Auxiliary Power Core
+        16539,  # Micro B88 Core Augmentation
+        16541,  # Micro K-Exhaust Core Augmentation
+    ],
+    31936: [  # Navy Micro Auxiliary Power Core
+        16543,  # Micro 'Vigor' Core Augmentation
+    ],
+    8089: [  # Compact Light Missile Launcher
+       8093,  # Prototype 'Arbalest' Light Missile Launcher
+    ],
+    8091: [  # Ample Light Missile Launcher
+        7993,  # Experimental TE-2100 Light Missile Launcher
+    ],
+    # Surface Cargo Scanner I was removed from game, however no mention of
+    # replacement module in patch notes. Morphing it to meta 0 module to be safe
+    442: [  # Cargo Scanner I
+        6129,  # Surface Cargo Scanner I
+    ]
+}
+
+def upgrade(saveddata_engine):
+    # Update fits schema
+    saveddata_engine.execute("ALTER TABLE fits ADD COLUMN targetResistsID INTEGER;")
+
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.iteritems():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
+

pyfa.py

@@ -84,14 +84,23 @@
     import os.path
 
     import eos.db
+    import eos.db.migration as migration
     import service.prefetch
     from gui.mainFrame import MainFrame
 
     #Make sure the saveddata db exists
     if not os.path.exists(config.savePath):
         os.mkdir(config.savePath)
 
-    eos.db.saveddata_meta.create_all()
+    if os.path.isfile(config.saveDB):
+        # If database exists, run migration after init'd database
+        eos.db.saveddata_meta.create_all()
+        migration.update(eos.db.saveddata_engine)
+    else:
+        # If database does not exist, do not worry about migration. Simply
+        # create and set version
+        eos.db.saveddata_meta.create_all()
+        eos.db.saveddata_engine.execute('PRAGMA user_version = %d'%config.dbversion)
 
     pyfa = wx.App(False)
     MainFrame()