MDL-15635 dtl refactoring and basic export and transfer ui (the impor…

…t UI will be part of install process soon) - based largely on code by Andrei Bautu - thanks!

admin/dbtransfer/database_export_form.php

@@ -0,0 +1,15 @@
+<?php //$Id$
+
+require_once $CFG->libdir.'/formslib.php';
+
+class database_export_form extends moodleform {
+
+    function definition() {
+        $mform = $this->_form;
+
+        $mform->addElement('header', 'database', get_string('dbexport', 'dbtransfer'));
+        $mform->addElement('textarea', 'description', get_string('description'), array('rows'=>5, 'cols'=>60));
+
+        $this->add_action_buttons(false, get_string('exportdata', 'dbtransfer'));
+    }
+}

admin/dbtransfer/database_transfer_form.php

@@ -0,0 +1,47 @@
+<?php //$Id$
+
+require_once $CFG->libdir.'/formslib.php';
+
+class database_transfer_form extends moodleform {
+
+    function definition() {
+        $mform = $this->_form;
+
+        $mform->addElement('header', 'database', get_string('dbtransfer', 'dbtransfer'));
+
+        $supported = array (
+            'mysqli/adodb',
+            'mysql/adodb',
+            'postgres7/adodb',
+            'mssql_n/adodb',
+            'mssql/adodb',
+            'odbc_mssql/adodb',
+            'oci8po/adodb',
+            'sqlite3/pdo',
+        );
+        $drivers = array();
+        foreach($supported as $driver) {
+            list($dbtype, $dblibrary) = explode('/', $driver);
+            $targetdb = moodle_database::get_driver_instance($dbtype, $dblibrary);
+            if ($targetdb->driver_installed() !== true) {
+                continue;
+            }
+            $drivers[$driver] = $driver;
+        }
+
+        $mform->addElement('select', 'driver', get_string('dbtype', 'install'), $drivers);
+        $mform->addElement('text', 'dbhost', get_string('dbhost', 'install'));
+        $mform->addElement('text', 'dbname', get_string('database', 'install'));
+        $mform->addElement('text', 'dbuser', get_string('user'));
+        $mform->addElement('text', 'dbpass', get_string('password'));
+        $mform->addElement('text', 'prefix', get_string('dbprefix', 'install'));
+
+        $mform->addRule('dbhost', get_string('required'), 'required', null);
+        $mform->addRule('dbname', get_string('required'), 'required', null);
+        $mform->addRule('dbuser', get_string('required'), 'required', null);
+        $mform->addRule('dbpass', get_string('required'), 'required', null);
+        $mform->addRule('prefix', get_string('required'), 'required', null);
+
+        $this->add_action_buttons(false, get_string('transferdata', 'dbtransfer'));
+    }
+}

admin/dbtransfer/dbexport.php

@@ -0,0 +1,21 @@
+<?php  //$Id$
+
+require('../../config.php');
+require_once('lib.php');
+require_once('database_export_form.php');
+
+require_login();
+admin_externalpage_setup('dbexport');
+
+//create form
+$form = new database_export_form();
+
+if ($data = $form->get_data()) {
+    dbtransfer_export_xml_database($data->description, $DB);
+    die;
+}
+
+admin_externalpage_print_header();
+// TODO: add some more info here
+$form->display();
+admin_externalpage_print_footer();

admin/dbtransfer/index.php

@@ -0,0 +1,34 @@
+<?php  //$Id$
+
+require('../../config.php');
+require_once('lib.php');
+require_once('database_transfer_form.php');
+
+require_login();
+admin_externalpage_setup('dbtransfer');
+
+//create form
+$form = new database_transfer_form();
+
+if ($data = $form->get_data()) {
+    list($dbtype, $dblibrary) = explode('/', $data->driver);
+    $targetdb = moodle_database::get_driver_instance($dbtype, $dblibrary);
+    if (!$targetdb->connect($data->dbhost, $data->dbuser, $data->dbpass, $data->dbname, false, $data->prefix, null)) {
+        throw new dbtransfer_exception('notargetconectexception', null, "$CFG->wwwroot/$CFG->admin/dbtransfer/");
+    }
+    if ($targetdb->get_tables()) {
+        // TODO add exception or string...
+        error('Sorry, tables already exist in selected database. Can not continue.'); 
+    }
+    admin_externalpage_print_header();
+    dbtransfer_transfer_database($DB, $targetdb);
+    notify(get_string('success'), 'notifysuccess');
+    print_continue("$CFG->wwwroot/$CFG->admin/");
+    admin_externalpage_print_footer();
+    die;
+}
+
+admin_externalpage_print_header();
+// TODO: add some more info here
+$form->display();
+admin_externalpage_print_footer();

admin/dbtransfer/lib.php

@@ -0,0 +1,52 @@
+<?php  //$Id$
+
+/*
+
+TODO:
+  - exporting to server file >2GB fails in 32bit operating systems - needs warning
+  - we may run out of disk space exporting to srever file - we must verify the file is not truncated; read from the end of file?
+  - when sending file >4GB - FAT32 limit, Apache limit, browser limit - needs warning
+  - there must be some form of progress bar during export, transfer - new tracking class could be passed around
+  - command line operation - could work around some 2G/4G limits in PHP; useful for cron full backups
+  - by default allow exporting into empty database only (no tables with the same prefix yet)
+  - all dangerous operation (like deleting of all data) should be confirmed by key found in special file in dataroot
+    (user would need file access to dataroot which might prevent various "accidents")
+  - implement "Export/import running" notification in lib/setup.php (similar to new upgrade flag in config table)
+  - gzip compression when storing xml file - the xml is very verbose and full of repeated tags (zip is not suitable here at all)
+    this could help us keep the files bellow 2G (expected ratio is > 10:1)
+
+*/
+
+require_once($CFG->libdir.'/adminlib.php');
+require_once($CFG->libdir.'/dtllib.php');
+
+
+function dbtransfer_export_xml_database($description, $mdb) {
+    @set_time_limit(0);
+
+    session_write_close(); // release session
+
+    header('Content-Type: application/xhtml+xml');
+    header('Content-Disposition: attachment; filename=database.xml');
+    header('Expires: 0');
+    header('Cache-Control: must-revalidate,post-check=0,pre-check=0');
+    header('Pragma: public');
+
+    while(@ob_flush());
+
+    $var = new file_xml_database_exporter('php://output', $mdb);
+    $var->export_database($description);
+
+    // no more output
+    die;
+}
+
+
+function dbtransfer_transfer_database($sourcedb, $targetdb) {
+    @set_time_limit(0);
+
+    session_write_close(); // release session
+
+    $var = new database_mover($sourcedb, $targetdb);
+    $var->export_database(null);
+}

admin/settings/misc.php

@@ -5,10 +5,12 @@
 if ($hassiteconfig) { // speedup for non-admins, add all caps used on this page
 
     // Experimental settings page
-    $temp = new admin_settingpage('experimental', get_string('experimental', 'admin'));
+    $ADMIN->add('misc', new admin_category('experimental', get_string('experimental','admin')));
+
+    $temp = new admin_settingpage('experimentalsettings', get_string('experimentalsettings', 'admin'));
     $temp->add(new admin_setting_configcheckbox('enableglobalsearch', get_string('enableglobalsearch', 'admin'), get_string('configenableglobalsearch', 'admin'), 0));
     $temp->add(new admin_setting_configcheckbox('smartpix', get_string('smartpix', 'admin'), get_string('configsmartpix', 'admin'), 0));
-    $item = new admin_setting_configcheckbox('enablehtmlpurifier', get_string('enablehtmlpurifier', 'admin'), get_string('configenablehtmlpurifier', 'admin'), 0); 
+    $item = new admin_setting_configcheckbox('enablehtmlpurifier', get_string('enablehtmlpurifier', 'admin'), get_string('configenablehtmlpurifier', 'admin'), 0);
     $item->set_updatedcallback('reset_text_filters_cache');
     $temp->add($item);
 
@@ -17,7 +19,11 @@
     $temp->add(new admin_setting_configcheckbox('enablecompletion', get_string('enablecompletion','completion'), get_string('configenablecompletion','completion'), COMPLETION_DISABLED));
     $temp->add(new admin_setting_pickroles('progresstrackedroles', get_string('progresstrackedroles','completion'), get_string('configprogresstrackedroles', 'completion'), array('moodle/legacy:student')));
 
-    $ADMIN->add('misc', $temp);
+    $ADMIN->add('experimental', $temp);
+
+    // DB transfer related pages
+    $ADMIN->add('experimental', new admin_externalpage('dbtransfer', get_string('dbtransfer', 'dbtransfer'), $CFG->wwwroot.'/'.$CFG->admin.'/dbtransfer/index.php', 'moodle/site:config', false));
+    $ADMIN->add('experimental', new admin_externalpage('dbexport', get_string('dbexport', 'dbtransfer'), $CFG->wwwroot.'/'.$CFG->admin.'/dbtransfer/dbexport.php', 'moodle/site:config', false));
 
     // XMLDB editor
     $ADMIN->add('misc', new admin_externalpage('xmldbeditor', get_string('xmldbeditor'), "$CFG->wwwroot/$CFG->admin/xmldb/"));

lang/en_utf8/admin.php

@@ -361,6 +361,7 @@
 $string['errorsetting'] = 'Could not save setting:';
 $string['errorwithsettings'] = 'Some settings were not changed due to an error.';
 $string['experimental'] = 'Experimental';
+$string['experimentalsettings'] = 'Experimental settings';
 $string['extendedusernamechars'] = 'Allow extended characters in usernames';
 $string['filecreated'] = 'New file created';
 $string['filestoredin'] = 'Save file into folder :';

lang/en_utf8/dbtransfer.php

@@ -0,0 +1,13 @@
+<?php // $Id$
+
+$string['dbexport'] = 'Database export';
+$string['dbtransfer'] = 'Database transfer';
+$string['differenttableexception'] = 'Table $a structure does not match.';
+$string['exportdata'] = 'Export data';
+$string['exportschemaexception'] = 'Current database structure does not match all install.xml files. <br /> $a';
+$string['importschemaexception'] = 'Current database structure does not match all install.xml files. <br /> $a';
+$string['importversionmismatchexception'] = 'Current version $->currentver does match exported version $a->schemaver.';
+$string['malformedxmlexception'] = 'Malformed XML found, can not continue.';
+$string['notargetconectexception'] = 'Can not connect target database, sorry.';
+$string['transferdata'] = 'Transfer data';
+$string['unknowntableexception'] = 'Unknown table $a found in export file.';

lib/dtl/database_exporter.php

@@ -104,25 +104,31 @@ public abstract function export_table_data(xmldb_table $table, $data);
      * @see $check_schema is true), queries the database and calls
      * appropiate callbacks.
      *
-     * @exception export_exception if any checking (e.g. database schema) fails
+     * @exception dbtransfer_exception if any checking (e.g. database schema) fails
      *
      * @param string $description a user description of the data.
      */
     public function export_database($description=null) {
         global $CFG;
 
-        if ($this->check_schema and $this->manager->check_database_schema($this->schema)) {
-            //TODO put message in error lang
-            throw new export_exception('XMLDB schema does not match database schema.');
+        if ($this->check_schema and $errors = $this->manager->check_database_schema($this->schema)) {
+            $details = '';
+            foreach ($errors as $table=>$items) {
+                $details .= '<div>'.get_string('table').' '.$table.':';
+                $details .= '<ul>';
+                foreach ($items as $item) {
+                    $details .= '<li>'.$item.'</li>';
+                }
+                $details .= '</ul></div>';
+            }
+            throw new dbtransfer_exception('exportschemaexception', $details);
         }
         $tables = $this->schema->getTables();
         $this->begin_database_export($CFG->version, $CFG->release, date('c'), $description);
         foreach ($tables as $table) {
             $rs = $this->mdb->get_recordset_sql('SELECT * FROM {'.$table->getName().'}');
-            //TODO remove this when dml will have exceptions
             if (!$rs) {
-                //TODO put message in error lang
-                throw new export_exception('An error occured while reading the database.');
+                throw new ddl_table_missing_exception($table->getName());
             }
             $this->begin_table_export($table);
             foreach ($rs as $row) {

lib/dtl/database_importer.php

@@ -57,7 +57,7 @@ public function __construct(moodle_database $mdb, $check_schema=true) {
      * operation, before any database changes are made. It will check the database
      * schema if @see check_schema is true
      *
-     * @exception import_exception if any checking (e.g. database schema, Moodle
+     * @exception dbtransfer_exception if any checking (e.g. database schema, Moodle
      * version) fails
      *
      * @param float $version the version of the system which generated the data
@@ -73,13 +73,21 @@ public function begin_database_import($version, $timestamp) {
         }
 
         if (round($version, 2) !== round($CFG->version, 2)) { // version might be in decimal format too
-            //TODO put message in error lang
-            throw new import_exception('Current Moodle version does not match exported Moodle version.');
+            $a = (object)array('schemaver'=>$version, 'currentver'=>$CFG->version);
+            throw new dbtransfer_exception('importversionmismatchexception', $a);
         }
 
-        if ($this->check_schema && $this->manager->check_database_schema($this->schema)) {
-            //TODO put message in error lang
-            throw new import_exception('XMLDB schema does not match database schema.');
+        if ($this->check_schema and $errors = $this->manager->check_database_schema($this->schema)) {
+            $details = '';
+            foreach ($errors as $table=>$items) {
+                $details .= '<div>'.get_string('table').' '.$table.':';
+                $details .= '<ul>';
+                foreach ($items as $item) {
+                    $details .= '<li>'.$item.'</li>';
+                }
+                $details .= '</ul></div>';
+            }
+            throw new dbtransfer_exception('importschemaexception', $details);
         }
         $this->mdb->begin_sql();
     }
@@ -88,7 +96,7 @@ public function begin_database_import($version, $timestamp) {
      * Callback function. Should be called only once per table import operation,
      * before any table changes are made. It will delete all table data.
      *
-     * @exception import_exception an unknown table import is attempted
+     * @exception dbtransfer_exception an unknown table import is attempted
      * @exception ddl_table_missing_exception if the table is missing
      *
      * @param string $tablename - the name of the table that will be imported
@@ -97,16 +105,13 @@ public function begin_database_import($version, $timestamp) {
      */
     public function begin_table_import($tablename, $schemaHash) {
         if (!$table = $this->schema->getTable($tablename)) {
-            //TODO put message in error lang
-            throw new import_exception('Unknown table in import data');
+            throw new dbtransfer_exception('unknowntableexception', $tablename);
         }
         if ($schemaHash != $table->getHash()) {
-            throw new import_exception('XMLDB schema does not match database schema.');
+            throw new dbtransfer_exception('differenttableexception', $tablename);
         }
         // this should not happen, unless someone drops tables after import started
         if (!$this->manager->table_exists($table)) {
-            // in the future, missing tables will be recreated with
-            //$this->manager->create_table($table);
             throw new ddl_table_missing_exception($tablename);
         }
         $this->mdb->delete_records($tablename);

lib/dtl/database_mover.php

@@ -7,15 +7,15 @@ class database_mover extends database_exporter {
     /**
      * Object constructor.
      *
-     * @param moodle_database $mdb_target Connection to the target database (a
-     * @see moodle_database object).
      * @param moodle_database $mdb Connection to the source database (a
      * @see moodle_database object).
+     * @param moodle_database $mdb_target Connection to the target database (a
+     * @see moodle_database object).
      * @param boolean $check_schema - whether or not to check that XML database
      * schema matches the RDBMS database schema before exporting (used by
      * @see export_database).
      */
-    public function __construct(moodle_database $mdb_target, moodle_database $mdb_source, $check_schema=true) {
+    public function __construct(moodle_database $mdb_source, moodle_database $mdb_target, $check_schema=true) {
         parent::__construct($mdb_source, $check_schema);
         $this->importer = new database_importer($mdb_target, $check_schema);
     }

lib/dtl/file_xml_database_exporter.php

@@ -12,17 +12,14 @@ class file_xml_database_exporter extends xml_database_exporter {
     /**
      * Object constructor.
      *
-     * @param string $filepath - path to the XML data file. Use null for PHP
+     * @param string $filepath - path to the XML data file. Use 'php://output' for PHP
      * output stream.
      * @param moodle_database $mdb Connection to the source database
      * @see xml_database_exporter::__construct()
      * @param boolean $check_schema - whether or not to check that XML database
      * @see xml_database_exporter::__construct()
      */
     public function __construct($filepath, moodle_database $mdb, $check_schema=true) {
-        if (is_null($filepath)) {
-            $filepath = 'php://output';
-        }
         parent::__construct($mdb, $check_schema);
         $this->filepath = $filepath;
     }
@@ -38,7 +35,7 @@ protected function output($text) {
      * Specific implementation for file exporting the database: it opens output stream, calls
      * superclass @see database_exporter::export_database() and closes output stream.
      *
-     * @exception export_exception if any checking (e.g. database schema) fails
+     * @exception dbtransfer_exception if any checking (e.g. database schema) fails
      *
      * @param string $description a user description of the data.
      */