Skip to content

GitLab

Commit 15b384bb authored by Robert Lyon's avatar Robert Lyon Committed by Gerrit Code Review
Browse files

Merge "Bug#1237198: Make Elasticsearch plugin work with MySQL."

parents d0c4a5a0 ca2d6dbb
Show whitespace changes
Inline Side-by-side
htdocs/search/elasticsearch/lang/en.utf8/search.elasticsearch.php
View file @ 15b384bb
......@@ -46,8 +46,6 @@ $string['Media'] = 'Media';
$string['noticeenabled'] = 'The Elasticsearch plugin is currently active. To deactivate it, deselect it in the <a href="%s">site options search settings</a>.';
$string['noticenotactive'] = 'The ElasticSearch Server is unreachable on host: %s and port %s. Please make sure it is running.';
$string['noticenotenabled'] = 'The Elasticsearch plugin is not currently enabled. To activate it, select it in the <a href="%s">site options in the search settings</a>.';
$string['noticepostgresrequired'] = 'The Elasticsearch plugin only works with a PostgreSQL database at this time.';
$string['noticepostgresrequiredtitle'] = 'Feature not available';
$string['owner'] = 'Owner';
$string['page'] = 'Page';
$string['pages'] = 'Pages';
......
htdocs/search/elasticsearch/lib.php
View file @ 15b384bb
......@@ -76,8 +76,7 @@ class PluginSearchElasticsearch extends PluginSearch {
* as the sitewide search plugin (i.e. get_config('searchplugin'))
*/
public static function is_available_for_site_setting() {
// It only works in postgres, because of the triggers
return is_postgres();
return true;
}
/**
......@@ -132,23 +131,6 @@ class PluginSearchElasticsearch extends PluginSearch {
public static function get_config_options() {
if (!is_postgres()) {
$smarty = smarty_core();
$smarty->assign('header', get_string('noticepostgresrequiredtitle', 'search.elasticsearch'));
$smarty->assign('notice', get_string('noticepostgresrequired', 'search.elasticsearch'));
$requirepostgreshtml = $smarty->fetch('Search:elasticsearch:configwarning.tpl');
unset($smarty);
$config = array(
'elements' => array(
'requirespostgres' => array(
'type' => 'html',
'value' => $requirepostgreshtml,
)
)
);
return $config;
}
if (get_config('searchplugin') == 'elasticsearch') {
$smarty = smarty_core();
$smarty->assign('notice', get_string('noticeenabled', 'search.elasticsearch', get_config('wwwroot').'admin/site/options.php?fs=searchsettings'));
......@@ -297,10 +279,10 @@ class PluginSearchElasticsearch extends PluginSearch {
if (count($types) > 0) {
$item_by_type_in_queue = array();
$rs = get_records_sql_array('SELECT type, count(*) FROM {search_elasticsearch_queue} GROUP BY type', array());
$rs = get_records_sql_array('SELECT type, count(*) AS total FROM {search_elasticsearch_queue} GROUP BY type', array());
if ($rs) {
foreach ($rs as $record) {
$item_by_type_in_queue[$record->type] = $record->count;
$item_by_type_in_queue[$record->type] = $record->total;
}
}
......@@ -344,15 +326,8 @@ class PluginSearchElasticsearch extends PluginSearch {
}
public function save_config_options($values) {
// Plugin only supports postgres. Don't bother saving anything if not Postgres.
if (!is_postgres()) {
return true;
}
set_config_plugin('search', 'elasticsearch', 'cronlimit', $values['cronlimit']);
// Changes in artefact types:
// - we need to add the newly selected artefact types (for indexing)
// - we need to removed artefact types that have been unchecked (to remove them from the index)
......@@ -965,6 +940,9 @@ abstract class ElasticsearchType
protected $item_to_index;
protected $mapping;
private static $mysqltriggeroperations = array('insert', 'update', 'delete');
/**
* $conditions was originally used to filter only active, non deleted records to insert into the queue,
* but as we will now use it to determine if the record has to be indexed or removed from the index.
......@@ -1098,6 +1076,7 @@ abstract class ElasticsearchType
*/
public static function create_triggers() {
$type = $this::$type;
if (is_postgres()) {
$sql = "DROP FUNCTION IF EXISTS {search_elasticsearch_{$type}}() CASCADE;";
execute_sql($sql);
......@@ -1105,6 +1084,23 @@ abstract class ElasticsearchType
FOR EACH ROW EXECUTE PROCEDURE {search_elasticsearch_queue_trigger}()";
execute_sql($sql);
}
else {
foreach (self::$mysqltriggeroperations as $operation) {
$sql = "DROP TRIGGER IF EXISTS {search_elasticsearch_{$type}_{$operation}};";
execute_sql($sql);
$oldid = ($operation == 'insert' ? 'null' : 'OLD.id');
$newid = ($operation == 'delete' ? 'null' : 'NEW.id');
$tablename = get_config('dbprefix') . ($type);
// For inserts, the NEW.id is not available until AFTER the record is insereted.
$triggertime = ($operation == 'insert' ? 'AFTER' : 'BEFORE');
$sql = "CREATE TRIGGER {search_elasticsearch_{$type}_{$operation}} {$triggertime} {$operation} ON {{$type}}
FOR EACH ROW CALL {search_elasticsearch_queue_trigger}('{$tablename}', '{$operation}', {$oldid}, {$newid})";
execute_sql($sql);
}
}
}
}
/**
......@@ -1354,6 +1350,9 @@ class ElasticsearchPseudotype_all
*/
class ElasticsearchIndexing {
// mysql tables require a trigger for each operation - insert, update, delete.
private static $mysqltriggeroperations = array('insert', 'update', 'delete');
/**
* Creates the Index on the elasticsearch server itself, first dropping if it already
* exists.
......@@ -1461,11 +1460,7 @@ class ElasticsearchIndexing {
* CASCADE, this will also delete the triggers which use these functions
*/
public static function drop_trigger_functions() {
// The trigger function syntax is postgresql specific
if (!is_postgres()) {
return;
}
if (is_postgres()) {
$sql = 'DROP FUNCTION IF EXISTS {search_elasticsearch_queue_trigger}() CASCADE;';
execute_sql($sql);
......@@ -1475,6 +1470,17 @@ class ElasticsearchIndexing {
$sql = 'DROP FUNCTION IF EXISTS {search_elasticsearch_queue2_trigger}() CASCADE;';
execute_sql($sql);
}
else {
$sql = 'DROP PROCEDURE IF EXISTS {search_elasticsearch_queue_trigger};';
execute_sql($sql);
$sql = 'DROP PROCEDURE IF EXISTS {search_elasticsearch_queue_artefact_trigger};';
execute_sql($sql);
$sql = 'DROP PROCEDURE IF EXISTS {search_elasticsearch_queue2_trigger};';
execute_sql($sql);
}
}
/**
* Creates trigger functions used by elasticsearch. These detect indexable records being inserted, deleted, or updated, and put
......@@ -1488,12 +1494,6 @@ class ElasticsearchIndexing {
* triggers you want to retain, after you call this function.
*/
public static function create_trigger_functions() {
// The trigger and function syntax is postgresql specific
if (!is_postgres()) {
return;
}
// Delete the trigger functions first.
// NOTE: This also deletes ALL the elasticsearch triggers.
self::drop_trigger_functions();
......@@ -1502,12 +1502,13 @@ class ElasticsearchIndexing {
// We'll use this to trim the prefix from table names before inserting them into
// search_elasticsearch_queue.type
$prefixlength = strlen(get_config('dbprefix'));
$dbprefix = get_config('dbprefix');
$prefixlength = strlen($dbprefix);
if ($prefixlength) {
$tablewithoutprefix = "RIGHT(TG_TABLE_NAME, -{$prefixlength})";
$tablewithoutprefix = (is_postgres() ? "RIGHT(TG_TABLE_NAME, -{$prefixlength})" : "SUBSTRING(tablename, " . ($prefixlength + 1) . ")");
}
else {
$tablewithoutprefix= 'TG_TABLE_NAME';
$tablewithoutprefix= (is_postgres() ? 'TG_TABLE_NAME' : "tablename");
}
//----------------------------------------------------------------------------------------------------
......@@ -1516,6 +1517,7 @@ class ElasticsearchIndexing {
// - Set it to monitor the table the type is named after
// - On an INSERT, UPDATE, or DELETE, just inserts the record ID and the name of the type in the queue table
// - When you modify a view, you also insert a record for every artefact in that view
if (is_postgres()) {
$sql = 'CREATE FUNCTION {search_elasticsearch_queue_trigger}() RETURNS trigger AS $search_elasticsearch_queue_trigger$
BEGIN
IF (TG_OP=\'DELETE\') THEN
......@@ -1549,6 +1551,40 @@ class ElasticsearchIndexing {
END IF;
END;
$search_elasticsearch_queue_trigger$ LANGUAGE plpgsql;';
}
else {
$sql = 'CREATE PROCEDURE {search_elasticsearch_queue_trigger}
(tablename varchar(64), operation varchar(10), oldid bigint, newid bigint)
BEGIN
IF (operation=\'delete\') THEN
IF NOT EXISTS (SELECT 1 FROM {search_elasticsearch_queue} WHERE itemid = oldid AND type = '.$tablewithoutprefix.') THEN
INSERT INTO {search_elasticsearch_queue} (itemid, type) VALUES (oldid, '.$tablewithoutprefix.');
END IF;
IF (tablename=\'' . $dbprefix . 'view\') THEN
INSERT INTO {search_elasticsearch_queue} (itemid, type)
SELECT va.artefact, \'artefact\' AS type
FROM {view_artefact} va
INNER JOIN {artefact} a ON va.artefact = a.id
WHERE va.view = oldid
AND va.artefact NOT IN (SELECT itemid FROM {search_elasticsearch_queue} WHERE type = \'artefact\')
AND a.artefacttype IN ' . $artefacttypes_str .';
END IF;
ELSE
IF NOT EXISTS (SELECT 1 FROM {search_elasticsearch_queue} WHERE itemid = newid AND type = ' . $tablewithoutprefix . ') THEN
INSERT INTO {search_elasticsearch_queue} (itemid, type) VALUES (newid, ' . $tablewithoutprefix . ');
END IF;
IF (tablename=\'' . $dbprefix . 'view\') THEN
INSERT INTO {search_elasticsearch_queue} (itemid, type)
SELECT va.artefact, \'artefact\' AS type
FROM {view_artefact} va
INNER JOIN {artefact} a ON va.artefact = a.id
WHERE va.view = newid
AND va.artefact NOT IN (SELECT itemid FROM {search_elasticsearch_queue} WHERE type = \'artefact\')
AND a.artefacttype IN ' . $artefacttypes_str .';
END IF;
END IF;
END';
}
execute_sql($sql);
//----------------------------------------------------------------------------------------------------
......@@ -1557,6 +1593,7 @@ class ElasticsearchIndexing {
// - Set it to monitor the artefact table
// - The main difference from the search_elasticsearch_queue_trigger, is that it also populates the
// artefacttype column in the queue table.
if (is_postgres()) {
$sql = 'CREATE FUNCTION {search_elasticsearch_queue_artefact_trigger}() RETURNS trigger AS $search_elasticsearch_queue_artefact_trigger$
BEGIN
IF (TG_OP=\'DELETE\') THEN
......@@ -1572,12 +1609,29 @@ class ElasticsearchIndexing {
END IF;
END;
$search_elasticsearch_queue_artefact_trigger$ LANGUAGE plpgsql;';
}
else {
$sql = 'CREATE PROCEDURE {search_elasticsearch_queue_artefact_trigger}
(tablename varchar(64), operation varchar(10), oldid bigint, oldartefacttype varchar(255), newid bigint, newartefacttype varchar(255))
BEGIN
IF (operation=\'delete\') THEN
IF (oldartefacttype IN ' . $artefacttypes_str . ') AND NOT EXISTS (SELECT 1 FROM {search_elasticsearch_queue} WHERE itemid = oldid AND type = ' . $tablewithoutprefix . ') THEN
INSERT INTO {search_elasticsearch_queue} (itemid, type, artefacttype) VALUES (oldid, ' . $tablewithoutprefix . ', oldartefacttype);
END IF;
ELSE
IF (newartefacttype IN ' . $artefacttypes_str . ') AND NOT EXISTS (SELECT 1 FROM {search_elasticsearch_queue} WHERE itemid = newid AND type = ' . $tablewithoutprefix . ') THEN
INSERT INTO {search_elasticsearch_queue} (itemid, type, artefacttype) VALUES (newid, ' . $tablewithoutprefix . ', newartefacttype);
END IF;
END IF;
END';
}
execute_sql($sql);
//----------------------------------------------------------------------------------------------------
// search_elasticsearch_queue2_trigger
// - For the view_artefact table
// - Whenever that table is modified, add a record into the queue table for the artefact mentioned
if (is_postgres()) {
$sql = 'CREATE FUNCTION {search_elasticsearch_queue2_trigger}() RETURNS trigger AS $search_elasticsearch_queue2_trigger$
BEGIN
IF (TG_OP=\'DELETE\') THEN
......@@ -1593,6 +1647,22 @@ class ElasticsearchIndexing {
END IF;
END;
$search_elasticsearch_queue2_trigger$ LANGUAGE plpgsql;';
}
else {
$sql = 'CREATE PROCEDURE {search_elasticsearch_queue2_trigger}
(tablename varchar(64), operation varchar(10), oldartefact bigint, newartefact bigint)
BEGIN
IF (operation=\'delete\') THEN
IF NOT EXISTS (SELECT 1 FROM {search_elasticsearch_queue} WHERE itemid = oldartefact AND type = \'artefact\') THEN
INSERT INTO {search_elasticsearch_queue} (itemid, type) VALUES (oldartefact, \'artefact\');
END IF;
ELSE
IF NOT EXISTS (SELECT 1 FROM {search_elasticsearch_queue} WHERE itemid = newartefact AND type = \'artefact\') THEN
INSERT INTO {search_elasticsearch_queue} (itemid, type) VALUES (newartefact, \'artefact\');
END IF;
END IF;
END';
}
execute_sql($sql);
}
......@@ -1604,6 +1674,7 @@ class ElasticsearchIndexing {
public static function drop_triggers($type) {
// The artefact type uses different trigger functions than the other types
if ($type == 'artefact') {
if (is_postgres()) {
$sql = "DROP FUNCTION IF EXISTS {search_elasticsearch_{$type}}() CASCADE;";
execute_sql($sql);
......@@ -1611,9 +1682,29 @@ class ElasticsearchIndexing {
execute_sql($sql);
}
else {
// need to drop 3 triggers for INSERT, UPDATE and DELETE
foreach (self::$mysqltriggeroperations as $operation) {
$sql = "DROP TRIGGER IF EXISTS {search_elasticsearch_{$type}_{$operation}};";
execute_sql($sql);
$sql = "DROP TRIGGER IF EXISTS {search_elasticsearch_view_artefact_{$operation}};";
execute_sql($sql);
}
}
}
else {
if (is_postgres()) {
$sql = "DROP FUNCTION IF EXISTS {search_elasticsearch_{$type}}() CASCADE;";
execute_sql($sql);
}
else {
// need to drop 3 triggers for INSERT, UPDATE and DELETE
foreach (self::$mysqltriggeroperations as $operation) {
$sql = "DROP TRIGGER IF EXISTS {search_elasticsearch_{$type}_{$operation}}";
execute_sql($sql);
}
}
}
}
/**
......@@ -1627,6 +1718,7 @@ class ElasticsearchIndexing {
self::drop_triggers($type);
// The artefact type uses different trigger functions than the other types
if ($type == 'artefact') {
if (is_postgres()) {
$sql = "CREATE TRIGGER {search_elasticsearch_{$type}} BEFORE INSERT OR UPDATE OR DELETE ON {{$type}}
FOR EACH ROW EXECUTE PROCEDURE {search_elasticsearch_queue_artefact_trigger}()";
execute_sql($sql);
......@@ -1636,12 +1728,56 @@ class ElasticsearchIndexing {
execute_sql($sql);
}
else {
foreach (self::$mysqltriggeroperations as $operation) {
$oldid = ($operation == 'insert' ? 'null' : 'OLD.id');
$newid = ($operation == 'delete' ? 'null' : 'NEW.id');
$oldartefacttype = ($operation == 'insert' ? 'null' : 'OLD.artefacttype');
$newartefacttype = ($operation == 'delete' ? 'null' : 'NEW.artefacttype');
$oldartefact = ($operation == 'insert' ? 'null' : 'OLD.artefact');
$newartefact = ($operation == 'delete' ? 'null' : 'NEW.artefact');
// For inserts, the NEW.id is not available until AFTER the record is insereted.
$triggertime = ($operation == 'insert' ? 'AFTER' : 'BEFORE');
// To remove confusion, include the table prefix if it exists as we'll be
// passing the actual table name to the stored procedure.
$tablename = get_config('dbprefix') . $type;
$viewtablename = get_config('dbprefix') . 'view_artefact';
// create 3 triggers on the artefact table.
$sql = "CREATE TRIGGER {search_elasticsearch_{$type}_{$operation}} {$triggertime} {$operation} ON {{$type}}
FOR EACH ROW CALL {search_elasticsearch_queue_artefact_trigger}('{$tablename}', '{$operation}', {$oldid}, {$oldartefacttype}, {$newid}, {$newartefacttype})";
execute_sql($sql);
// create 3 triggers on the view_artefact table.
$sql = "CREATE TRIGGER {search_elasticsearch_view_artefact_{$operation}} {$triggertime} {$operation} ON {view_artefact}
FOR EACH ROW CALL {search_elasticsearch_queue2_trigger}('{$viewtablename}', '{$operation}', {$oldartefact}, {$newartefact})";
execute_sql($sql);
}
}
}
else {
if (is_postgres()) {
$sql = "CREATE TRIGGER {search_elasticsearch_{$type}} BEFORE INSERT OR UPDATE OR DELETE ON {{$type}}
FOR EACH ROW EXECUTE PROCEDURE {search_elasticsearch_queue_trigger}()";
execute_sql($sql);
}
else {
// create the 3 triggers on the table.
foreach (self::$mysqltriggeroperations as $operation) {
$oldid = ($operation == 'insert' ? 'null' : 'OLD.id');
$newid = ($operation == 'delete' ? 'null' : 'NEW.id');
// For inserts, the NEW.id is not available until AFTER the record is insereted.
$triggertime = ($operation == 'insert' ? 'AFTER' : 'BEFORE');
// To remove confusion, include the table prefix if it exists as we'll be
// passing the actual table name to the stored procedure.
$tablename = get_config('dbprefix') . ($type);
$sql = "CREATE TRIGGER {search_elasticsearch_{$type}_{$operation}} {$triggertime} {$operation} ON {{$type}}
FOR EACH ROW CALL {search_elasticsearch_queue_trigger}('{$tablename}', '{$operation}', {$oldid}, {$newid})";
execute_sql($sql);
}
}
}
}
public static function artefacttypes_filter_string() {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment