MDL-35773 Backup: API should have option to not backup files

Allow both UI and automated backups to be created without including files. Instead include only file references. This is essentially implementing "SAMESITE" to backup files instead of only for import and export functionality. A new backup setting to include files (defaults to yes) has been included. The restore process will also look for and attempt to restore files from the trashdir as part of restoring backups. Additionally to support this process the ammount of time files are kept in trashdir before they are cleaned up via cron is also adjustable via admin setting.
safatshahin · Jul 26, 2019 · d7e4481 · d7e4481
1 parent f622ee9
commit d7e4481
Show file tree

Hide file tree

Showing 17 changed files with 190 additions and 18 deletions.
diff --git a/admin/settings/courses.php b/admin/settings/courses.php
@@ -206,6 +206,11 @@
     $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_role_assignments', new lang_string('generalroleassignments','backup'), new lang_string('configgeneralroleassignments','backup'), array('value'=>1, 'locked'=>0)));
     $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_activities', new lang_string('generalactivities','backup'), new lang_string('configgeneralactivities','backup'), array('value'=>1, 'locked'=>0)));
     $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_blocks', new lang_string('generalblocks','backup'), new lang_string('configgeneralblocks','backup'), array('value'=>1, 'locked'=>0)));
+    $temp->add(new admin_setting_configcheckbox_with_lock(
+            'backup/backup_general_files',
+            new lang_string('generalfiles', 'backup'),
+            new lang_string('configgeneralfiles', 'backup'),
+            array('value' => '1', 'locked' => 0)));
     $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_filters', new lang_string('generalfilters','backup'), new lang_string('configgeneralfilters','backup'), array('value'=>1, 'locked'=>0)));
     $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_comments', new lang_string('generalcomments','backup'), new lang_string('configgeneralcomments','backup'), array('value'=>1, 'locked'=>0)));
     $temp->add(new admin_setting_configcheckbox_with_lock('backup/backup_general_badges', new lang_string('generalbadges','backup'), new lang_string('configgeneralbadges','backup'), array('value'=>1,'locked'=>0)));
@@ -342,6 +347,10 @@
     $temp->add(new admin_setting_configcheckbox('backup/backup_auto_activities', new lang_string('generalactivities','backup'), new lang_string('configgeneralactivities','backup'), 1));
     $temp->add(new admin_setting_configcheckbox('backup/backup_auto_blocks', new lang_string('generalblocks','backup'), new lang_string('configgeneralblocks','backup'), 1));
     $temp->add(new admin_setting_configcheckbox('backup/backup_auto_filters', new lang_string('generalfilters','backup'), new lang_string('configgeneralfilters','backup'), 1));
+    $temp->add(new admin_setting_configcheckbox(
+            'backup/backup_auto_files',
+            new lang_string('generalfiles', 'backup'),
+            new lang_string('configgeneralfiles', 'backup'), '1'));
     $temp->add(new admin_setting_configcheckbox('backup/backup_auto_comments', new lang_string('generalcomments','backup'), new lang_string('configgeneralcomments','backup'), 1));
     $temp->add(new admin_setting_configcheckbox('backup/backup_auto_badges', new lang_string('generalbadges','backup'), new lang_string('configgeneralbadges','backup'), 1));
     $temp->add(new admin_setting_configcheckbox('backup/backup_auto_calendarevents', new lang_string('generalcalendarevents','backup'), new lang_string('configgeneralcalendarevents','backup'), 1));

diff --git a/admin/settings/server.php b/admin/settings/server.php
@@ -172,7 +172,10 @@
 
 $ADMIN->add('server', $temp);
 
-
+    $temp->add(new admin_setting_configduration('filescleanupperiod',
+        new lang_string('filescleanupperiod', 'admin'),
+        new lang_string('filescleanupperiod_help', 'admin'),
+        86400));
 
 $ADMIN->add('server', new admin_externalpage('environment', new lang_string('environment','admin'), "$CFG->wwwroot/$CFG->admin/environment.php"));
 $ADMIN->add('server', new admin_externalpage('phpinfo', new lang_string('phpinfo'), "$CFG->wwwroot/$CFG->admin/phpinfo.php"));

diff --git a/backup/backup.php b/backup/backup.php
@@ -150,6 +150,13 @@
     $loghtml = '';
     if ($backup->get_stage() == backup_ui::STAGE_FINAL) {
 
+        // Before we perform the backup check settings to see if user
+        // or setting defaults are set to exclude files from the backup.
+        if ($backup->get_setting_value('files') == 0) {
+            $bc->set_mode(backup::MODE_SAMESITE);
+            $renderer->set_samesite_notification();
+        }
+
         if ($backupmode != backup::MODE_ASYNC) {
             // Synchronous backup handling.
 
@@ -180,6 +187,7 @@
             // Hide the progress display and first backup step bar (the 'finished' step will show next).
             echo html_writer::end_div();
             echo html_writer::script('document.getElementById("executionprogress").style.display = "none";');
+
         } else {
             // Async backup handling.
             $backup->get_controller()->finish_ui();
@@ -203,6 +211,8 @@
                     'restoreurl' => $restoreurl->out(),
                     'headingident' => 'backup'
             );
+
+            echo $renderer->set_samesite_notification();
             echo $renderer->render_from_template('core/async_backup_status', $progresssetup);
         }
 

diff --git a/backup/controller/backup_controller.class.php b/backup/controller/backup_controller.class.php
@@ -185,6 +185,20 @@ public function process_ui_event() {
         backup_check::check_security($this, false);
     }
 
+    /**
+     * Sets the mode (purpose) of the backup.
+     *
+     * @param int $mode The mode to set.
+     */
+    public function set_mode($mode) {
+        $this->mode = $mode;
+        $this->set_include_files(); // Need to check if files are included as mode may have changed.
+        $this->save_controller();
+        $tbc = self::load_controller($this->backupid);
+        $this->logger = $tbc->logger; // Wakeup loggers.
+        $tbc->plan->destroy(); // Clean plan controller structures, keeping logger alive.
+    }
+
     public function set_status($status) {
         // Note: never save_controller() with the object info after STATUS_EXECUTING or the whole controller,
         // containing all the steps will be sent to DB. 100% (monster) useless.
@@ -410,6 +424,13 @@ protected function set_include_files() {
             $includefiles = false;
         }
 
+        // If backup is automated and we have set auto backup config to exclude
+        // files then set them to be excluded here.
+        $backupautofiles = (bool)get_config('backup', 'backup_auto_files');
+        if ($this->get_mode() === backup::MODE_AUTOMATED && !$backupautofiles) {
+            $includefiles = false;
+        }
+
         $this->includefiles = (int) $includefiles;
         $this->log("setting file inclusion to {$this->includefiles}", backup::LOG_DEBUG);
         return $this->includefiles;

diff --git a/backup/moodle2/backup_root_task.class.php b/backup/moodle2/backup_root_task.class.php
@@ -110,6 +110,12 @@ protected function define_settings() {
         $this->add_setting($blocks);
         $this->converter_deps($blocks, $converters);
 
+        // Define files.
+        $files = new backup_generic_setting('files', base_setting::IS_BOOLEAN, true);
+        $files->set_ui(new backup_setting_ui_checkbox($files, get_string('rootsettingfiles', 'backup')));
+        $this->add_setting($files);
+        $this->converter_deps($files, $converters);
+
         // Define filters
         $filters = new backup_generic_setting('filters', base_setting::IS_BOOLEAN, true);
         $filters->set_ui(new backup_setting_ui_checkbox($filters, get_string('rootsettingfilters', 'backup')));

diff --git a/backup/util/dbops/backup_plan_dbops.class.php b/backup/util/dbops/backup_plan_dbops.class.php
@@ -201,7 +201,8 @@ public static function get_mnet_localhost_wwwroot() {
     * @param bool $useidonly only use the ID in the file name
     * @return string The filename to use
     */
-    public static function get_default_backup_filename($format, $type, $id, $users, $anonymised, $useidonly = false) {
+    public static function get_default_backup_filename($format, $type, $id, $users, $anonymised,
+            $useidonly = false, $files = true) {
         global $DB;
 
         // Calculate backup word
@@ -251,6 +252,11 @@ public static function get_default_backup_filename($format, $type, $id, $users,
             $info = '-an';
         }
 
+        // Indicate if backup doesn't contain files.
+        if (!$files) {
+            $info .= '-nf';
+        }
+
         return $backupword . '-' . $format . '-' . $type . '-' .
                $name . '-' . $date . $info . '.mbz';
     }

diff --git a/backup/util/dbops/restore_dbops.class.php b/backup/util/dbops/restore_dbops.class.php
@@ -1054,17 +1054,25 @@ public static function send_files_to_pool($basepath, $restoreid, $component, $fi
                     // Create the file in the filepool if it does not exist yet.
                     if (!$fs->file_exists($newcontextid, $component, $filearea, $rec->newitemid, $file->filepath, $file->filename)) {
 
-                        // Even if a file has been deleted since the backup was made, the file metadata will remain in the
-                        // files table, and the file will not be moved to the trashdir.
-                        // Files are not cleared from the files table by cron until several days after deletion.
+                        // Even if a file has been deleted since the backup was made, the file metadata may remain in the
+                        // files table, and the file will not yet have been moved to the trashdir. e.g. a draft file version.
+                        // Try to recover from file table first.
                         if ($foundfiles = $DB->get_records('files', array('contenthash' => $file->contenthash), '', '*', 0, 1)) {
                             // Only grab one of the foundfiles - the file content should be the same for all entries.
                             $foundfile = reset($foundfiles);
                             $fs->create_file_from_storedfile($file_record, $foundfile->id);
                         } else {
-                            // A matching existing file record was not found in the database.
-                            $results[] = self::get_missing_file_result($file);
-                            continue;
+                            // Finally try to restore the file from trash.
+                            $filesytem = $fs->get_file_system();
+                            $restorefile = $file;
+                            $restorefile->contextid = $newcontextid;
+                            $storedfile = new stored_file($fs, $restorefile);
+                            $trashrecovery = $filesytem->recover_file($storedfile, true);
+                            if (!$trashrecovery) {
+                                // A matching file was not found.
+                                $results[] = self::get_missing_file_result($file);
+                                continue;
+                            }
                         }
                     }
                 }

diff --git a/backup/util/helper/backup_cron_helper.class.php b/backup/util/helper/backup_cron_helper.class.php
@@ -415,8 +415,9 @@ public static function launch_automated_backup($course, $starttime, $userid) {
             $id = $bc->get_id();
             $users = $bc->get_plan()->get_setting('users')->get_value();
             $anonymised = $bc->get_plan()->get_setting('anonymize')->get_value();
+            $incfiles = (bool)$config->backup_auto_files;
             $bc->get_plan()->get_setting('filename')->set_value(backup_plan_dbops::get_default_backup_filename($format, $type,
-                    $id, $users, $anonymised));
+                    $id, $users, $anonymised, false, $incfiles));
 
             $bc->set_status(backup::STATUS_AWAITING);
 

diff --git a/backup/util/helper/backup_helper.class.php b/backup/util/helper/backup_helper.class.php
@@ -287,7 +287,15 @@ static public function store_backup_file($backupid, $filepath, \core\progress\ba
             $config = get_config('backup');
             $dir = $config->backup_auto_destination;
             if ($config->backup_auto_storage == 1 and $dir and is_dir($dir) and is_writable($dir)) {
-                $filedest = $dir.'/'.backup_plan_dbops::get_default_backup_filename($format, $backuptype, $courseid, $hasusers, $isannon, !$config->backup_shortname);
+                $filedest = $dir.'/'
+                        .backup_plan_dbops::get_default_backup_filename(
+                                $format,
+                                $backuptype,
+                                $courseid,
+                                $hasusers,
+                                $isannon,
+                                !$config->backup_shortname,
+                                (bool)$config->backup_auto_files);
                 // first try to move the file, if it is not possible copy and delete instead
                 if (@rename($filepath, $filedest)) {
                     return null;

diff --git a/backup/util/ui/backup_ui_stage.class.php b/backup/util/ui/backup_ui_stage.class.php
@@ -155,7 +155,9 @@ public function get_next_stage() {
                             $this->ui->get_type(),
                             $this->ui->get_controller_id(),
                             $this->ui->get_setting_value('users'),
-                            $this->ui->get_setting_value('anonymize')
+                            $this->ui->get_setting_value('anonymize'),
+                            false,
+                            (bool)$this->ui->get_setting_value('files')
                         );
                         $setting->set_value($filename);
                     }
@@ -457,7 +459,16 @@ protected function initialise_stage_form() {
                         $id = $this->ui->get_controller_id();
                         $users = $this->ui->get_setting_value('users');
                         $anonymised = $this->ui->get_setting_value('anonymize');
-                        $setting->set_value(backup_plan_dbops::get_default_backup_filename($format, $type, $id, $users, $anonymised));
+                        $files = (bool)$this->ui->get_setting_value('files');
+                        $filename = backup_plan_dbops::get_default_backup_filename(
+                                $format,
+                                $type,
+                                $id,
+                                $users,
+                                $anonymised,
+                                false,
+                                $files);
+                        $setting->set_value($filename);
                     }
                     $form->add_setting($setting, $task);
                     break;
@@ -628,6 +639,7 @@ public function display(core_backup_renderer $renderer) {
         if (!empty($this->results['missing_files_in_pool'])) {
             $output .= $renderer->notification(get_string('missingfilesinpool', 'backup'), 'notifyproblem');
         }
+        $output .= $renderer->get_samesite_notification();
         $output .= $renderer->notification(get_string('executionsuccess', 'backup'), 'notifysuccess');
         $output .= $renderer->continue_button($restorerul);
         $output .= $renderer->box_end();

diff --git a/backup/util/ui/renderer.php b/backup/util/ui/renderer.php
@@ -43,6 +43,13 @@
  */
 class core_backup_renderer extends plugin_renderer_base {
 
+    /**
+     * Same site notification display.
+     *
+     * @var string
+     */
+    private $samesitenotification = '';
+
     /**
      * Renderers a progress bar for the backup or restore given the items that make it up.
      *
@@ -80,6 +87,22 @@ public function log_display($loghtml) {
         return $out;
     }
 
+    /**
+     * Set the same site backup notification.
+     *
+     */
+    public function set_samesite_notification() {
+        $this->samesitenotification = $this->output->notification(get_string('samesitenotification', 'backup'), 'info');
+    }
+
+    /**
+     * Get the same site backup notification.
+     *
+     */
+    public function get_samesite_notification() {
+        return $this->samesitenotification;
+    }
+
     /**
      * Prints a dependency notification
      *

diff --git a/lang/en/admin.php b/lang/en/admin.php
@@ -564,6 +564,8 @@
 $string['extendedusernamechars'] = 'Allow extended characters in usernames';
 $string['extramemorylimit'] = 'Extra PHP memory limit';
 $string['fatalsessionautostart'] = '<p>Serious configuration error detected, please notify server administrator.</p><p> To operate properly, Moodle requires that administrator changes PHP settings.</p><p><code>session.auto_start</code> must be set to <code>off</code>.</p><p>This setting is controlled by editing <code>php.ini</code>, Apache/IIS <br />configuration or <code>.htaccess</code> file on the server.</p>';
+$string['filescleanupperiod'] = 'Clean trash pool files';
+$string['filescleanupperiod_help'] = 'How often trash files are removed. These are files that are associated with a context that no longer exists';
 $string['fileconversioncleanuptask'] = 'Cleanup of temporary records for file conversions.';
 $string['filecreated'] = 'New file created';
 $string['filestoredin'] = 'Save file into folder :';

diff --git a/lang/en/backup.php b/lang/en/backup.php
@@ -127,6 +127,7 @@
 $string['configgeneralcalendarevents'] = 'Sets the default for including calendar events in a backup.';
 $string['configgeneralcomments'] = 'Sets the default for including comments in a backup.';
 $string['configgeneralcompetencies'] = 'Sets the default for including competencies in a backup.';
+$string['configgeneralfiles'] = 'Sets the default for including files in a backup.';
 $string['configgeneralfilters'] = 'Sets the default for including filters in a backup.';
 $string['configgeneralhistories'] = 'Sets the default for including user history within a backup.';
 $string['configgenerallogs'] = 'If enabled logs will be included in backups by default.';
@@ -202,6 +203,7 @@
 $string['generalcomments'] = 'Include comments';
 $string['generalcompetencies'] = 'Include competencies';
 $string['generalenrolments'] = 'Include enrolment methods';
+$string['generalfiles'] = 'Include files';
 $string['generalfilters'] = 'Include filters';
 $string['generalhistories'] = 'Include histories';
 $string['generalgradehistories'] = 'Include histories';
@@ -328,6 +330,7 @@
 $string['rootsettingblocks'] = 'Include blocks';
 $string['rootsettingcompetencies'] = 'Include competencies';
 $string['rootsettingfilters'] = 'Include filters';
+$string['rootsettingfiles'] = 'Include files';
 $string['rootsettingcomments'] = 'Include comments';
 $string['rootsettingcalendarevents'] = 'Include calendar events';
 $string['rootsettinguserscompletion'] = 'Include user completion details';
@@ -337,6 +340,7 @@
 $string['rootsettinggroups'] = 'Include groups and groupings';
 $string['rootsettingimscc1'] = 'Convert to IMS Common Cartridge 1.0';
 $string['rootsettingimscc11'] = 'Convert to IMS Common Cartridge 1.1';
+$string['samesitenotification'] = 'This backup was created with only references to files, not the files themselves. Restoring will only work on this site.';
 $string['sitecourseformatwarning'] = 'This is a front page backup, note that they can only be restored on the front page';
 $string['storagecourseonly'] = 'Course backup filearea';
 $string['storagecourseandexternal'] = 'Course backup filearea and the specified directory';

diff --git a/lib/filestorage/file_storage.php b/lib/filestorage/file_storage.php
@@ -2241,7 +2241,8 @@ public function cron() {
 
         // remove trash pool files once a day
         // if you want to disable purging of trash put $CFG->fileslastcleanup=time(); into config.php
-        if (empty($CFG->fileslastcleanup) or $CFG->fileslastcleanup < time() - 60*60*24) {
+        $filescleanupperiod = empty($CFG->filescleanupperiod) ? 86400 : $CFG->filescleanupperiod;
+        if (empty($CFG->fileslastcleanup) || ($CFG->fileslastcleanup < time() - $filescleanupperiod)) {
             require_once($CFG->libdir.'/filelib.php');
             // Delete files that are associated with a context that no longer exists.
             mtrace('Cleaning up files from deleted contexts... ', '');

diff --git a/lib/filestorage/file_system.php b/lib/filestorage/file_system.php
@@ -236,6 +236,15 @@ abstract public function copy_content_from_storedfile(stored_file $file, $target
      */
     abstract public function remove_file($contenthash);
 
+    /**
+     * Tries to recover missing content of file from trash.
+     *
+     * @param stored_file $file stored_file instance
+     * @param bool $createrecord Create file record for stored file.
+     * @return bool success
+     */
+    abstract public function recover_file(stored_file $file, $createrecord=false);
+
     /**
      * Check whether a file is removable.
      *