[core] consolidate database opening workflow

Before this change, we've tried to open the database in read/write, but not create mode. In situations where the database didn't exist yet, this logged an error to the console, and we proceeded to opening it again in read/write/create mode, which actually created the file. The reason we did this is so that we could detect really old caching databases from January 2016 in case a developer upgraded from an older SDK (iOS v3.1 and earlier, Android v3.2 and earlier) that didn't have #3715 yet.

However, these error messages, while innocent, look scary in the console and some users suspect that it's a bug. This change opens the file directly in read/write/create mode, omitting the first failed attempt. To handle old cache databases, we're now deleting the `http_cache` table, which was the only table in those old databases, and create the new schema, rather than deleting the entire file and recreating the Database object. In most scenarios, this will lead to one fewer opening attempt, while the database migration will continue to work for the few users who upgrade all the way from a January 2016 SDK.

Additionally, this fixes a mismatch between the Qt and non-Qt implementation: Qt doesn't support opening a file in read/write mode without the create flag. This means that we've seen a different control flow on Qt compared to the non-Qt implementation when opening a database.

platform/default/mbgl/storage/offline_database.cpp

@@ -28,67 +28,70 @@ OfflineDatabase::~OfflineDatabase() {
 }
 
 void OfflineDatabase::ensureSchema() {
-    if (path != ":memory:") {
-        auto result = mapbox::sqlite::Database::tryOpen(path, mapbox::sqlite::ReadWrite);
-        if (result.is<mapbox::sqlite::Exception>()) {
-            const auto& ex = result.get<mapbox::sqlite::Exception>();
-            if (ex.code == mapbox::sqlite::ResultCode::NotADB) {
-                // Corrupted; blow it away.
-                removeExisting();
-            } else if (ex.code == mapbox::sqlite::ResultCode::CantOpen) {
-                // Doesn't exist yet.
-            } else {
-                Log::Error(Event::Database, "Unexpected error connecting to database: %s", ex.what());
-                throw ex;
-            }
+    auto result = mapbox::sqlite::Database::tryOpen(path, mapbox::sqlite::ReadWriteCreate);
+    if (result.is<mapbox::sqlite::Exception>()) {
+        const auto& ex = result.get<mapbox::sqlite::Exception>();
+        if (ex.code == mapbox::sqlite::ResultCode::NotADB) {
+            // Corrupted; blow it away.
+            removeExisting();
+            result = mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate);
         } else {
-            try {
-                db = std::make_unique<mapbox::sqlite::Database>(std::move(result.get<mapbox::sqlite::Database>()));
-                db->setBusyTimeout(Milliseconds::max());
-                db->exec("PRAGMA foreign_keys = ON");
-
-                switch (userVersion()) {
-                case 0:
-                case 1:
-                    // cache-only database; ok to delete
-                    removeExisting();
-                    break;
-                case 2:
-                    migrateToVersion3();
-                    // fall through
-                case 3:
-                case 4:
-                    migrateToVersion5();
-                    // fall through
-                case 5:
-                    migrateToVersion6();
-                    // fall through
-                case 6:
-                    // happy path; we're done
-                    return;
-                default:
-                    // downgrade, delete the database
-                    removeExisting();
-                    break;
-                }
-            } catch (const mapbox::sqlite::Exception& ex) {
-                // Unfortunately, SQLITE_NOTADB is not always reported upon opening the database.
-                // Apparently sometimes it is delayed until the first read operation.
-                if (ex.code == mapbox::sqlite::ResultCode::NotADB) {
-                    removeExisting();
-                } else {
-                    throw;
-                }
-            }
+            Log::Error(Event::Database, "Unexpected error connecting to database: %s", ex.what());
+            throw ex;
         }
     }
 
     try {
-        #include "offline_schema.cpp.include"
-
-        db = std::make_unique<mapbox::sqlite::Database>(mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWrite | mapbox::sqlite::Create));
+        assert(result.is<mapbox::sqlite::Database>());
+        db = std::make_unique<mapbox::sqlite::Database>(std::move(result.get<mapbox::sqlite::Database>()));
         db->setBusyTimeout(Milliseconds::max());
         db->exec("PRAGMA foreign_keys = ON");
+
+        switch (userVersion()) {
+        case 0:
+        case 1:
+            // Newly created database, or old cache-only database; remove old table if it exists.
+            removeOldCacheTable();
+            break;
+        case 2:
+            migrateToVersion3();
+            // fall through
+        case 3:
+        case 4:
+            migrateToVersion5();
+            // fall through
+        case 5:
+            migrateToVersion6();
+            // fall through
+        case 6:
+            // happy path; we're done
+            return;
+        default:
+            // downgrade, delete the database
+            removeExisting();
+            break;
+        }
+    } catch (const mapbox::sqlite::Exception& ex) {
+        // Unfortunately, SQLITE_NOTADB is not always reported upon opening the database.
+        // Apparently sometimes it is delayed until the first read operation.
+        if (ex.code == mapbox::sqlite::ResultCode::NotADB) {
+            removeExisting();
+        } else {
+            throw;
+        }
+    }
+
+    try {
+        #include "offline_schema.cpp.include"
+
+        // When downgrading the database, or when the database is corrupt, we've deleted the old database handle,
+        // so we need to reopen it.
+        if (!db) {
+            db = std::make_unique<mapbox::sqlite::Database>(mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate));
+            db->setBusyTimeout(Milliseconds::max());
+            db->exec("PRAGMA foreign_keys = ON");
+        }
+
         db->exec("PRAGMA auto_vacuum = INCREMENTAL");
         db->exec("PRAGMA journal_mode = DELETE");
         db->exec("PRAGMA synchronous = FULL");
@@ -117,6 +120,11 @@ void OfflineDatabase::removeExisting() {
     }
 }
 
+void OfflineDatabase::removeOldCacheTable() {
+    db->exec("DROP TABLE IF EXISTS http_cache");
+    db->exec("VACUUM");
+}
+
 void OfflineDatabase::migrateToVersion3() {
     db->exec("PRAGMA auto_vacuum = INCREMENTAL");
     db->exec("VACUUM");

platform/default/mbgl/storage/offline_database.hpp

@@ -70,6 +70,7 @@ class OfflineDatabase : private util::noncopyable {
     int userVersion();
     void ensureSchema();
     void removeExisting();
+    void removeOldCacheTable();
     void migrateToVersion3();
     void migrateToVersion5();
     void migrateToVersion6();

platform/default/sqlite3.hpp

@@ -11,9 +11,8 @@ namespace mapbox {
 namespace sqlite {
 
 enum OpenFlag : int {
-    ReadOnly = 0x00000001,
-    ReadWrite = 0x00000002,
-    Create = 0x00000004,
+    ReadOnly        = 0b001,
+    ReadWriteCreate = 0b110,
 };
 
 enum class ResultCode : int {

test/storage/offline_database.test.cpp

@@ -23,17 +23,6 @@ TEST(OfflineDatabase, TEST_REQUIRES_WRITE(Create)) {
     OfflineDatabase db(filename);
     EXPECT_FALSE(bool(db.get({ Resource::Unknown, "mapbox://test" })));
 
-#ifdef __QT__
-    // Qt doesn't support opening a SQLite database without also creating it if it doesn't exist yet.
-    // Therefore, we're currently using the code path that thinks that we opened an old database
-    // (user_version = 0), deletes and recreates the database.
-    EXPECT_EQ(1u, log.count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
-#endif
-#ifndef __QT__
-    // Only non-Qt platforms are setting a logger on the SQLite object.
-    EXPECT_EQ(1u, log.count({ EventSeverity::Info, Event::Database, static_cast<int64_t>(mapbox::sqlite::ResultCode::CantOpen), "cannot open file" }, true));
-    EXPECT_EQ(1u, log.count({ EventSeverity::Info, Event::Database, static_cast<int64_t>(mapbox::sqlite::ResultCode::CantOpen), "No such file or directory" }, true));
-#endif
     EXPECT_EQ(0u, log.uncheckedCount());
 }
 
@@ -42,13 +31,12 @@ TEST(OfflineDatabase, TEST_REQUIRES_WRITE(SchemaVersion)) {
     util::deleteFile(filename);
 
     {
-        mapbox::sqlite::Database db = mapbox::sqlite::Database::open(filename, mapbox::sqlite::Create | mapbox::sqlite::ReadWrite);
+        mapbox::sqlite::Database db = mapbox::sqlite::Database::open(filename, mapbox::sqlite::ReadWriteCreate);
         db.exec("PRAGMA user_version = 1");
     }
 
     OfflineDatabase db(filename);
 
-    EXPECT_EQ(1u, log.count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
     EXPECT_EQ(0u, log.uncheckedCount());
 }
 
@@ -339,17 +327,6 @@ TEST(OfflineDatabase, TEST_REQUIRES_WRITE(ConcurrentUse)) {
     util::deleteFile(filename);
 
     OfflineDatabase db1(filename);
-#ifdef __QT__
-    // Qt doesn't support opening a SQLite database without also creating it if it doesn't exist yet.
-    // Therefore, we're currently using the code path that thinks that we opened an old database
-    // (user_version = 0), deletes and recreates the database.
-    EXPECT_EQ(1u, log.count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
-#endif
-#ifndef __QT__
-    // Only non-Qt platforms are setting a logger on the SQLite object.
-    EXPECT_EQ(1u, log.count({ EventSeverity::Info, Event::Database, static_cast<int64_t>(mapbox::sqlite::ResultCode::CantOpen), "cannot open file" }, true));
-    EXPECT_EQ(1u, log.count({ EventSeverity::Info, Event::Database, static_cast<int64_t>(mapbox::sqlite::ResultCode::CantOpen), "No such file or directory" }, true));
-#endif
     EXPECT_EQ(0u, log.uncheckedCount());
 
     OfflineDatabase db2(filename);

test/storage/sqlite.test.cpp

@@ -8,7 +8,7 @@ using namespace mbgl;
 TEST(SQLite, Statement) {
     using namespace mbgl;
 
-    mapbox::sqlite::Database db = mapbox::sqlite::Database::open(":memory:", mapbox::sqlite::Create | mapbox::sqlite::ReadWrite);
+    mapbox::sqlite::Database db = mapbox::sqlite::Database::open(":memory:", mapbox::sqlite::ReadWriteCreate);
     db.exec("CREATE TABLE test (id INTEGER);");
 
     mapbox::sqlite::Statement stmt1{ db, "INSERT INTO test (id) VALUES (?1);" };
@@ -49,7 +49,7 @@ TEST(SQLite, TEST_REQUIRES_WRITE(TryOpen)) {
 
 TEST(SQLite, CloseDatabaseWithPendingTransaction) {
     auto db = std::make_unique<mapbox::sqlite::Database>(mapbox::sqlite::Database::open(
-        ":memory:", mapbox::sqlite::ReadWrite | mapbox::sqlite::Create));
+        ":memory:", mapbox::sqlite::ReadWriteCreate));
     mapbox::sqlite::Transaction transaction(*db);
     transaction.commit();
 }
@@ -58,7 +58,7 @@ TEST(SQLite, CloseMovedDatabaseWithPendingTransaction) {
     // Verifies that we can correctly commit a transaction even if we move the Database object to
     // another address.
     auto db1 = mapbox::sqlite::Database::open(":memory:",
-                                              mapbox::sqlite::ReadWrite | mapbox::sqlite::Create);
+                                              mapbox::sqlite::ReadWriteCreate);
     std::unique_ptr<mapbox::sqlite::Database> db2;
     mapbox::sqlite::Transaction transaction(db1);
     db2 = std::make_unique<mapbox::sqlite::Database>(std::move(db1));