Skip to content

Commit

Permalink
MDL-55356 core_search: Make indexing task/CLI do context requests
Browse files Browse the repository at this point in the history
  • Loading branch information
@sammarshallou
sammarshallou committed Oct 11, 2017
1 parent 4ba11aa commit 74b7a42
Show file tree
Hide file tree
Showing 4 changed files with 201 additions and 3 deletions.
19 changes: 17 additions & 2 deletions lib/classes/task/search_index_task.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,22 @@ public function execute() {
}
$globalsearch = \core_search\manager::instance();

// Indexing database records for modules + rich documents of forum.
$globalsearch->index(false, get_config('core', 'searchindextime'), new \text_progress_trace());
// Get total indexing time limit.
$timelimit = get_config('core', 'searchindextime');
$start = time();

// Do normal indexing.
$globalsearch->index(false, $timelimit, new \text_progress_trace());

// Do requested indexing (if any) for the rest of the time.
if ($timelimit != 0) {
$now = time();
$timelimit -= ($now - $start);
if ($timelimit <= 1) {
// There is hardly any time left, so don't try to do requests.
return;
}
}
$globalsearch->process_index_requests($timelimit, new \text_progress_trace());
}
}
74 changes: 74 additions & 0 deletions search/classes/manager.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1087,4 +1087,78 @@ public static function request_index(\context $context, $areaid = '') {
'timerequested' => time(), 'partialarea' => '', 'partialtime' => 0 ];
$DB->insert_record('search_index_requests', $newrecord);
}

/**
* Processes outstanding index requests. This will take the first item from the queue and
* process it, continuing until an optional time limit is reached.
*
* If there are no index requests, the function will do nothing.
*
* @param float $timelimit Time limit (0 = none)
* @param \progress_trace|null $progress Optional progress indicator
*/
public function process_index_requests($timelimit = 0.0, \progress_trace $progress = null) {
global $DB;

if (!$progress) {
$progress = new \null_progress_trace();
}

$complete = false;
$before = microtime(true);
if ($timelimit) {
$stopat = $before + $timelimit;
}
while (true) {
// Retrieve first request, using fully defined ordering.
$requests = $DB->get_records('search_index_requests', null,
'timerequested, contextid, searcharea',
'id, contextid, searcharea, partialarea, partialtime', 0, 1);
if (!$requests) {
// If there are no more requests, stop.
$complete = true;
break;
}
$request = reset($requests);

// Calculate remaining time.
$remainingtime = 0;
$beforeindex = microtime(true);
if ($timelimit) {
$remainingtime = $stopat - $beforeindex;
}

// Show a message before each request, indicating what will be indexed.
$context = \context::instance_by_id($request->contextid);
$contextname = $context->get_context_name();
if ($request->searcharea) {
$contextname .= ' (search area: ' . $request->searcharea . ')';
}
$progress->output('Indexing requested context: ' . $contextname);

// Actually index the context.
$result = $this->index_context($context, $request->searcharea, $remainingtime,
$progress, $request->partialarea, $request->partialtime);

// Work out shared part of message.
$endmessage = $contextname . ' (' . round(microtime(true) - $beforeindex, 1) . 's)';

// Update database table and continue/stop as appropriate.
if ($result->complete) {
// If we completed the request, remove it from the table.
$DB->delete_records('search_index_requests', ['id' => $request->id]);
$progress->output('Completed requested context: ' . $endmessage);
} else {
// If we didn't complete the request, store the partial details (how far it got).
$DB->update_record('search_index_requests', ['id' => $request->id,
'partialarea' => $result->startfromarea,
'partialtime' => $result->startfromtime]);
$progress->output('Ending requested context: ' . $endmessage);

// The time limit must have expired, so stop looping.
break;
}
}
}

}
17 changes: 16 additions & 1 deletion search/cli/indexer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,26 @@
$limitunderline = preg_replace('~.~', '=', $limitinfo);
echo "Running index of site$limitinfo\n";
echo "=====================$limitunderline\n";
$timelimit = (int)$options['timelimit'];
} else {
echo "Running full index of site\n";
echo "==========================\n";
$timelimit = 0;
}
$globalsearch->index(false, $options['timelimit'], new text_progress_trace());
$before = time();
$globalsearch->index(false, $timelimit, new text_progress_trace());

// Do specific index requests with the remaining time.
if ($timelimit) {
$timelimit -= (time() - $before);
// Only do index requests if there is a reasonable amount of time left.
if ($timelimit > 1) {
$globalsearch->process_index_requests($timelimit, new text_progress_trace());
}
} else {
$globalsearch->process_index_requests(0, new text_progress_trace());
}

} else {
echo "Running full reindex of site\n";
echo "============================\n";
Expand Down
94 changes: 94 additions & 0 deletions search/tests/manager_test.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -738,4 +738,98 @@ public function test_request_index() {
\core_search\manager::request_index($forum2ctx);
$this->assertEquals(4, $DB->count_records('search_index_requests'));
}

/**
* Tests the process_index_requests function.
*/
public function test_process_index_requests() {
global $DB;

$this->resetAfterTest();

$search = testable_core_search::instance();

// When there are no index requests, nothing gets logged.
$progress = new progress_trace_buffer(new text_progress_trace(), false);
$search->process_index_requests(0.0, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();
$this->assertEquals('', $out);

// Set up the course with 3 forums.
$generator = $this->getDataGenerator();
$course = $generator->create_course(['fullname' => 'TCourse']);
$forum1 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum1']);
$forum2 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum2']);
$forum3 = $generator->create_module('forum', ['course' => $course->id, 'name' => 'TForum3']);

// Hack the forums so they have different creation times.
$now = time();
$DB->set_field('forum', 'timemodified', $now - 3, ['id' => $forum1->id]);
$DB->set_field('forum', 'timemodified', $now - 2, ['id' => $forum2->id]);
$DB->set_field('forum', 'timemodified', $now - 1, ['id' => $forum3->id]);
$forum2time = $now - 2;

// Make 2 index requests.
$search::request_index(context_course::instance($course->id), 'mod_label-activity');
$this->waitForSecond();
$search::request_index(context_module::instance($forum1->cmid));

// Run with no time limit.
$search->process_index_requests(0.0, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();

// Check that it's done both areas.
$this->assertContains(
'Indexing requested context: Course: TCourse (search area: mod_label-activity)',
$out);
$this->assertContains(
'Completed requested context: Course: TCourse (search area: mod_label-activity)',
$out);
$this->assertContains('Indexing requested context: Forum: TForum1', $out);
$this->assertContains('Completed requested context: Forum: TForum1', $out);

// Check the requests database table is now empty.
$this->assertEquals(0, $DB->count_records('search_index_requests'));

// Request indexing the course a couple of times.
$search::request_index(context_course::instance($course->id), 'mod_forum-activity');
$search::request_index(context_course::instance($course->id), 'mod_forum-post');

// Do the processing again with a time limit and indexing delay. The time limit is too
// small; because of the way the logic works, this means it will index 2 activities.
$search->get_engine()->set_add_delay(0.2);
$search->process_index_requests(0.1, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();

// Confirm the right wrapper information was logged.
$this->assertContains(
'Indexing requested context: Course: TCourse (search area: mod_forum-activity)',
$out);
$this->assertContains('Stopping indexing due to time limit', $out);
$this->assertContains(
'Ending requested context: Course: TCourse (search area: mod_forum-activity)',
$out);

// Check the database table has been updated with progress.
$records = array_values($DB->get_records('search_index_requests', null, 'searcharea'));
$this->assertEquals('mod_forum-activity', $records[0]->partialarea);
$this->assertEquals($forum2time, $records[0]->partialtime);

// Run again and confirm it now finishes.
$search->process_index_requests(0.1, $progress);
$out = $progress->get_buffer();
$progress->reset_buffer();
$this->assertContains(
'Completed requested context: Course: TCourse (search area: mod_forum-activity)',
$out);
$this->assertContains(
'Completed requested context: Course: TCourse (search area: mod_forum-post)',
$out);

// Confirm table is now empty.
$this->assertEquals(0, $DB->count_records('search_index_requests'));
}
}

0 comments on commit 74b7a42

Please sign in to comment.