PXB-3221 : Assertion failure: page0cur.cc:1177:ib::fatal triggered du…

…ring prepare/or next server startup

Problem:
--------
If there are tables created in system tablespace and if ALTER ADD INDEX/DROP INDEX is executed before the
backup lock is taken, system tablespace could end up in corrupted state.

This is because this operation is not redologged and we are supposed to recopy the system tablespace files.
But we dont track system tablesapce, neither reopen and recopy them. Hence this issue.

Fix:
----
1. Track system tablespace in the list of tables tracked/backedup
2. Removing tracking for tables in system tablespace except of recopy. Other operations can be played via redolog
3. Reopen system tablespace and recopy them as ibdata1.new/ ibdata2.new

storage/innobase/fsp/fsp0sysspace.cc

@@ -942,6 +942,12 @@ dberr_t SysTablespace::open_or_create(bool is_temp, bool create_new_db,
     }
   }
 
+  // Add system tablespace for tracking purpose. We might have
+  // to recopy it
+  if (ddl_tracker && opt_lock_ddl == LOCK_DDL_REDUCED) {
+    ddl_tracker->add_table_from_ibd_scan(space->id, space->name);
+  }
+
   return (err);
 }
 #endif /* !UNIV_HOTBACKUP */

storage/innobase/xtrabackup/src/ddl_tracker.cc

@@ -82,6 +82,8 @@ void ddl_tracker_t::add_table_from_ibd_scan(const space_id_t space_id,
 
 void ddl_tracker_t::add_undo_tablespace(const space_id_t space_id,
                                         std::string name) {
+  ut_ad(fsp_is_undo_tablespace(space_id));
+
   Fil_path::normalize(name);
   std::lock_guard<std::mutex> lock(m_ddl_tracker_mutex);
   if (is_server_locked()) {
@@ -122,6 +124,7 @@ void ddl_tracker_t::add_missing_table(std::string path) {
   std::lock_guard<std::mutex> lock(m_ddl_tracker_mutex);
   missing_tables.insert(path);
 }
+
 void ddl_tracker_t::add_create_table_from_redo(const space_id_t space_id,
                                                lsn_t start_lsn,
                                                const char *name) {
@@ -130,6 +133,12 @@ void ddl_tracker_t::add_create_table_from_redo(const space_id_t space_id,
     return;
   }
 
+  // tables in system tablespace need not be tracked. They can be
+  // created via redo log
+  if (fsp_is_system_tablespace(space_id)) {
+    return;
+  }
+
   std::string new_space_name = name;
   Fil_path::normalize(new_space_name);
   if (Fil_path::has_prefix(new_space_name, Fil_path::DOT_SLASH)) {
@@ -146,6 +155,12 @@ void ddl_tracker_t::add_rename_table_from_redo(const space_id_t space_id,
                                                lsn_t start_lsn,
                                                const char *old_name,
                                                const char *new_name) {
+  // Rename of tables in system tablespace is only rename in DD. this is
+  // tracked via redo log
+  if (fsp_is_system_tablespace(space_id)) {
+    return;
+  }
+
   std::string old_space_name{old_name};
   std::string new_space_name{new_name};
 
@@ -177,6 +192,12 @@ void ddl_tracker_t::add_drop_table_from_redo(const space_id_t space_id,
     return;
   }
 
+  // DROP table in system tablespace is drop indexes. There is no file removal
+  // this is also tracked/redone from redo
+  if (fsp_is_system_tablespace(space_id)) {
+    return;
+  }
+
   std::string new_space_name{name};
   Fil_path::normalize(new_space_name);
   if (Fil_path::has_prefix(new_space_name, Fil_path::DOT_SLASH)) {
@@ -206,6 +227,12 @@ void ddl_tracker_t::add_renamed_table(const space_id_t &space_id,
     return;
   }
 
+  // rename tables in system tablespace is only rename in DD
+  // tracked via redo. Skip this
+  if (fsp_is_system_tablespace(space_id)) {
+    return;
+  }
+
   Fil_path::normalize(new_name);
   if (Fil_path::has_prefix(new_name, Fil_path::DOT_SLASH)) {
     new_name.erase(0, 2);
@@ -381,6 +408,8 @@ void ddl_tracker_t::handle_ddl_operations() {
   for (auto &table : recopy_tables) {
     if (tables_in_backup.find(table) != tables_in_backup.end()) {
       if (renames.find(table) != renames.end()) {
+        // We never create .del for ibdata*
+        ut_ad(!fsp_is_system_tablespace(table));
         backup_file_printf(
             convert_file_name(table, renames[table].first, ".ibd.del").c_str(),
             "%s", "");
@@ -408,6 +437,8 @@ void ddl_tracker_t::handle_ddl_operations() {
       continue;
     }
 
+    // We never create .del for ibdata*
+    ut_ad(!fsp_is_system_tablespace(table.first));
     backup_file_printf(
         convert_file_name(table.first, table.second, ".ibd.del").c_str(), "%s",
         "");
@@ -449,8 +480,55 @@ void ddl_tracker_t::handle_ddl_operations() {
       table = new_tables.erase(table);
       continue;
     }
-    std::tie(err, std::ignore) = fil_open_for_xtrabackup(
-        table->second, table->second.substr(0, table->second.length() - 4));
+
+    if (fsp_is_system_tablespace(table->first)) {
+      // open system tablespace for recopying
+      srv_sys_space.shutdown();
+
+      srv_sys_space.set_space_id(TRX_SYS_SPACE);
+
+      /* Create the filespace flags. */
+      ulint fsp_flags =
+          fsp_flags_init(univ_page_size, false, false, false, false);
+      srv_sys_space.set_flags(fsp_flags);
+
+      srv_sys_space.set_name("innodb_system");
+      srv_sys_space.set_path(srv_data_home);
+
+      /* Supports raw devices */
+      if (!srv_sys_space.parse_params(innobase_data_file_path, true,
+                                      xtrabackup_prepare)) {
+        xb::error() << "Cannot parse system tablespace params";
+        ut_ad(0);
+        return;
+      }
+      dberr_t err;
+      page_no_t sum_of_new_sizes;
+      lsn_t flush_lsn;
+
+      std::this_thread::sleep_for(std::chrono::milliseconds(200));
+
+      err = srv_sys_space.check_file_spec(false, 0);
+
+      if (err != DB_SUCCESS) {
+        xb::error() << "could not find data files at the specified datadir";
+        ut_ad(0);
+        return;
+      }
+
+      err = srv_sys_space.open_or_create(false, false, &sum_of_new_sizes,
+                                         &flush_lsn);
+
+      if (err != DB_SUCCESS) {
+        xb::error() << "could not reopen system tablespace";
+        ut_ad(0);
+        return;
+      }
+
+    } else {
+      std::tie(err, std::ignore) = fil_open_for_xtrabackup(
+          table->second, table->second.substr(0, table->second.length() - 4));
+    }
     table++;
   }
 

storage/innobase/xtrabackup/test/suites/lockless/system_tablespace.sh

@@ -0,0 +1,57 @@
+# PXB-3221 : Assertion failure: page0cur.cc:1177:ib::fatal triggered during prepare
+. inc/common.sh
+
+require_debug_pxb_version
+start_server
+
+$MYSQL $MYSQL_ARGS -Ns -e "CREATE TABLE t1(a INT, KEY k1(a)) TABLESPACE=innodb_system;" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 VALUES (1),(2),(3),(4)" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+
+innodb_wait_for_flush_all
+
+XB_ERROR_LOG=$topdir/backup.log
+BACKUP_DIR=$topdir/backup
+rm -rf $BACKUP_DIR
+xtrabackup --backup --lock-ddl=REDUCED --target-dir=$BACKUP_DIR \
+--debug-sync="ddl_tracker_before_lock_ddl" 2> >( tee $XB_ERROR_LOG)&
+
+job_pid=$!
+pid_file=$topdir/backup/xtrabackup_debug_sync
+wait_for_xb_to_suspend $pid_file
+xb_pid=`cat $pid_file`
+echo "backup pid is $job_pid"
+
+mysql -e "ALTER TABLE t1 ADD INDEX k2(a)" test
+mysql -e "ALTER TABLE t1 DROP INDEX k1" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+
+
+# Resume the xtrabackup process
+vlog "Resuming xtrabackup"
+kill -SIGCONT $xb_pid
+run_cmd wait $job_pid
+
+xtrabackup --prepare --target-dir=$BACKUP_DIR
+stop_server
+
+rm -rf $mysql_datadir/*
+xtrabackup --copy-back --target-dir=$BACKUP_DIR
+start_server
+#mysql -e "ALTER TABLE t1 DROP INDEX k1" test
+mysql -e "DELETE FROM t1 WHERE a = 2" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+$MYSQL $MYSQL_ARGS -Ns -e "INSERT INTO t1 SELECT * FROM t1" test
+mysql -e "DELETE FROM t1 WHERE a = 2" test
+
+stop_server
+
+rm -rf $mysql_datadir $BACKUP_DIR
+rm $XB_ERROR_LOG