Commit 1bbc75e1 authored by Robert Lyon's avatar Robert Lyon

Bug 1694171: Fixing up outstanding issues for v5.0

This patch will sit above Geralds one to sort out last minute things
including:

- upgrade compatibility
-- Will need to compare current elasticsearch version with compatible
version and alert about problems
-- Will allow one to set the shards/replicas as part of config
-- alert the cluster health (non green) status and unallocated replica
shards

- Allow empty search query to return all related results
- Allow collections to record / search on collection tags
- Index group access correctly
- Get correct results to display for each tab rather than all results
bunched together
- Allow highlight on description field

- Add a basic behat test
- removed the built in docs/ and tests/ dirs as well as the phpunit
test files

Change-Id: I09b4eaf502a8400499debde2ff1d2a5316f20fbf
Signed-off-by: Robert Lyon's avatarRobert Lyon <robertl@catalyst.net.nz>
parent ffc0c3ee
......@@ -459,6 +459,7 @@ $string['sharedwithme'] = 'Shared with me';
$string['titleanddescription'] = 'Title, description, tags';
$string['titleanddescriptionnotags'] = 'Title, description';
$string['titleanddescriptionandtagsandowner'] = 'Title, description, tags, owner';
$string['tagsonly'] = 'Tags only'; // for elasticsearch
$string['tagsonly1'] = 'Tags';
$string['sharedviewsdescription'] = 'This page lists the most recently modified or commented on pages that have been shared with you. They may have been shared with you directly, shared with friends of the owner, or shared with one of your groups.';
$string['sharedwith'] = 'Shared with';
......
......@@ -71,5 +71,26 @@ function xmldb_search_elasticsearch_upgrade($oldversion=0) {
log_warn(get_string('newindextype', 'search.elasticsearch', 'collection'), true, false);
}
}
if ($oldversion < 2017080300) {
if ($result = get_record_sql("SELECT si.version, si.release
FROM {search_installed} si
JOIN {config} c ON c.value = si.name
WHERE si.name = 'elasticsearch'
AND c.field = 'searchplugin'", array())) {
log_debug('Updating elasticsearch plugin to be compatible with Elasticsearch version 5');
// set the shards / replicas to default elasticsearch values
set_config_plugin('search', 'elasticsearch', 'shards', 5);
set_config_plugin('search', 'elasticsearch', 'replicashards', 1);
list($status, $info) = call_static_method(generate_class_name('search', 'elasticsearch'), 'elasticsearch_server');
if (!empty($info->error)) {
// warn them if problems with elasticsearch server
log_warn($info->error);
}
// warn them they will need to reindex site
log_warn(get_string('newversion', 'search.elasticsearch', PluginSearchElasticsearch::elasticsearchphp_version, PluginSearchElasticsearch::elasticsearch_version), true, false);
}
}
return true;
}
......@@ -48,6 +48,7 @@ try {
catch (ParameterException $e) {
json_reply('missingparameter','Missing parameter \'query\'');
}
$query = PluginSearchElasticsearch::clean_query($query);
$data = PluginSearchElasticsearch::search_all($query, $limit, $offset, $options, $mainfacetterm);
$data['query'] = $query;
......
......@@ -23,6 +23,7 @@ $string['atoz'] = 'A to Z';
$string['blogpost'] = 'Journal entry';
$string['bypassindexname'] = 'Bypass index';
$string['bypassindexnamedescription'] = '(Optional) If provided, Mahara will load index data into this index name instead of the main index name.';
$string['clusterstatus'] = 'There is a problem with the elasticsearch cluster. The status is "%s" and unallocated shards are "%s".';
$string['collection'] = 'Collection';
$string['confignotset'] = '(not set)';
$string['contains'] = 'Contains';
......@@ -35,6 +36,7 @@ $string['daterecentfirst'] = 'Date (most recent first)';
$string['deleted'] = 'Deleted';
$string['deletedforumpost'] = 'Deleted forum post';
$string['document'] = 'Document';
$string['elasticsearchtooold'] = 'Your version of elasticsearch "%s" is too old. It needs to be "%s" or higher';
$string['filterresultsby'] = 'Filter results by';
$string['forum'] = 'Forum';
$string['forumpost'] = 'Forum post';
......@@ -50,6 +52,7 @@ $string['indexnamedescription'] = 'Name of the Elasticsearch index. Default is m
$string['license'] = 'License';
$string['Media'] = 'Media';
$string['newindextype'] = 'A new index type "%s" has been added to your elasticsearch settings. For this to take effect you will need to reindex your site';
$string['newversion'] = 'A new Elasticsearch PHP version "%s" has been added to Mahara that is compatible with Elasticsearch server "%s" and above. For this to take effect you will need to reindex your site';
$string['none'] = 'none';
$string['noticeenabled'] = 'The Elasticsearch plugin is currently active. To deactivate it, deselect it in the <a href="%s">site options search settings</a>.';
$string['noticenotactive'] = 'The ElasticSearch Server is unreachable on host: %s and port %s. Please make sure it is running.';
......@@ -67,11 +70,15 @@ $string['Portfolio'] = 'Portfolio';
$string['record'] = 'record';
$string['records'] = 'records';
$string['relevance'] = 'Relevance';
$string['replicashards'] = 'Replica shards';
$string['replicashardsdescription'] = 'The number of copies of shards to be made. Note: if you only have 1 node then set replicas to 0';
$string['reset'] = 'Reset';
$string['resetallindexes'] = 'Reset ALL indexes';
$string['resetdescription'] = 'This table shows the number of records of each type currently in the queue to be sent to the Elasticsearch server. Items are sent to the Elasticsearch server each time the search plugin\'s cron task runs (every 5 minutes). Click on the button at the bottom to reset the search index, deleting all records and requeuing them.';
$string['resetlegend'] = 'Index reset';
$string['resume'] = 'Résumé';
$string['shards'] = 'Shards';
$string['shardsdescription'] = 'The number of pieces of the index to be made';
$string['sortby'] = 'Sort by';
$string['tags'] = 'Tags';
$string['tagsonly'] = 'Tags only';
......@@ -84,5 +91,6 @@ $string['username'] = 'Auth username';
$string['usernamedescription'] = '(Optional) Username to pass to Elasticsearch via HTTP basic auth';
$string['Users'] = 'Users';
$string['wallpost'] = 'Wall post';
$string['xsearchresultsfory'] = '%s search results for %s';
$string['xsearchresults'] = 'Displaying %s search results';
$string['xsearchresultsfory'] = 'Displaying %s search results for "%s"';
$string['ztoa'] = 'Z to A';
This diff is collapsed.
......@@ -80,36 +80,34 @@ class ElasticsearchType_artefact extends ElasticsearchType {
),
// array of groups that have access to the artefact - empty (all), member, admin
'groups' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'type' => 'object',
'include_in_all' => FALSE,
'copy_to' => 'group',
/*
// list of groups for which both members and admins have access to the artefact
'all' => array(
'type' => 'int',
'index' => 'not_analyzed',
'include_in_all' => FALSE
),
// list of groups for which only admins have access to the artefact
'admin' => array(
'type' => 'int',
'index' => 'not_analyzed',
'include_in_all' => FALSE
),
// list of groups for which only members have access to the artefact
'member' => array(
'type' => 'int',
'index' => 'not_analyzed',
'include_in_all' => FALSE
),
// list of groups for which only tutors have access to the artefact
'tutor' => array(
'type' => 'int',
'index' => 'not_analyzed',
'include_in_all' => FALSE
),
*/
'properties' => array (
'all' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'admin' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'member' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'tutor' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
)
)
),
'group' => array (
'type' => 'integer'
......@@ -397,11 +395,10 @@ class ElasticsearchType_artefact extends ElasticsearchType {
$record_views = array ();
foreach ( $views as $view ) {
if (isset ( $view->id )) {
$record_views [$view->id] = $view->title;
$record_views[$view->id] = $view->title;
}
}
$record_views = self::views_by_artefact_acl_filter ( $record_views );
$record_views = self::views_by_artefact_acl_filter($record_views);
$record->views = $record_views;
}
......@@ -432,10 +429,14 @@ class ElasticsearchType_artefact extends ElasticsearchType {
$size = intval ( $record->width * 80 / $record->height ) . 'x80';
}
}
$record->thumb = ArtefactTypeImage::get_icon ( array (
'id' => $id,
'size' => $size
) );
$vars = array (
'id' => $id,
'size' => $size
);
if (!empty($record->views)) {
$vars['viewid'] = key($record->views); // use first view we are can see
}
$record->thumb = ArtefactTypeImage::get_icon ($vars);
}
return $record;
......@@ -489,6 +490,7 @@ class ElasticsearchType_artefact extends ElasticsearchType {
global $USER;
$acl = new ElasticsearchFilterAcl ( $USER );
$filter = [
"bool" => [
"must" => [
......@@ -523,11 +525,13 @@ class ElasticsearchType_artefact extends ElasticsearchType {
)
);
/*
* $pretty = json_encode ( $params, JSON_PRETTY_PRINT );
* var_dump_error ( $pretty );
*/
$results = $client->search ( $params );
$valid = array();
if (!empty($results['hits'])) {
foreach($results['hits']['hits'] as $item) {
$valid[$item['_source']['id']] = $views[$item['_source']['id']];
}
}
return $valid;
}
}
......@@ -60,10 +60,34 @@ class ElasticsearchType_block_instance extends ElasticsearchType {
),
// array of groups that have access to the artefact - empty (all), member, admin, tutor
'groups' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => FALSE
'type' => 'object',
'include_in_all' => FALSE,
'properties' => array (
'all' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'admin' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'member' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'tutor' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
)
)
),
'group' => array (
'type' => 'integer'
......
......@@ -24,6 +24,14 @@ class ElasticsearchType_collection extends ElasticsearchType {
'type' => 'text',
'include_in_all' => TRUE
),
'tags' => array (
'type' => 'keyword',
'copy_to' => 'tag',
'include_in_all' => TRUE
),
'tag' => array (
'type' => 'keyword'
),
// the owner can be owner (user), group, or institution
'owner' => array (
'type' => 'long',
......@@ -64,10 +72,34 @@ class ElasticsearchType_collection extends ElasticsearchType {
),
// array of groups that have access to the artefact
'groups' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
'type' => 'object',
'include_in_all' => FALSE,
'properties' => array (
'all' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'admin' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'member' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'tutor' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
)
)
),
'group' => array (
'type' => 'integer'
......@@ -107,6 +139,7 @@ class ElasticsearchType_collection extends ElasticsearchType {
'id' => NULL,
'name' => NULL,
'description' => NULL,
'tags' => NULL,
'owner' => NULL,
'group' => NULL,
'institution' => NULL,
......@@ -122,7 +155,15 @@ class ElasticsearchType_collection extends ElasticsearchType {
if (! $record) {
return false;
}
$tags = get_records_array ('collection_tag', 'collection', $id);
if ($tags != false) {
foreach ($tags as $tag) {
$record->tags [] = $tag->tag;
}
}
else {
$record->tags = null;
}
// Access: get view_access info
$access = self::collection_access_records ( $id );
$accessObj = self::access_process ( $access );
......@@ -179,7 +220,16 @@ class ElasticsearchType_collection extends ElasticsearchType {
}
$record->views = $record_views;
}
// Tags
$tags = get_records_array('collection_tag', 'collection', $id);
if ($tags != false) {
foreach ($tags as $tag) {
$record->tags [] = $tag->tag;
}
}
else {
$record->tags = null;
}
return $record;
}
......
......@@ -40,10 +40,16 @@ class ElasticsearchType_group extends ElasticsearchType {
'include_in_all' => FALSE
),
'groups' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
'type' => 'object',
'include_in_all' => FALSE,
'properties' => array (
'member' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
)
)
),
'group' => array (
'type' => 'integer'
......
......@@ -30,10 +30,34 @@ class ElasticsearchType_interaction_forum_post extends ElasticsearchType {
'include_in_all' => FALSE,
'properties' => array (
'groups' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
'type' => 'object',
'include_in_all' => FALSE,
'properties' => array (
'all' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'admin' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'member' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'tutor' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
)
)
),
'group' => array (
'type' => 'integer'
......
......@@ -35,10 +35,34 @@ class ElasticsearchType_interaction_instance extends ElasticsearchType {
'include_in_all' => FALSE
),
'groups' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
'type' => 'object',
'include_in_all' => FALSE,
'properties' => array (
'all' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'admin' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'member' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'tutor' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
)
)
),
'group' => array (
'type' => 'integer'
......
......@@ -73,41 +73,35 @@ class ElasticsearchType_view extends ElasticsearchType {
),
// array of groups that have access to the artefact - empty (all), member, admin, tutor
'groups' => array (
// 'type' => 'object',
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => FALSE,
/*
// list of groups for which both members and admins have access to the artefact
'all' => array (
'type' => 'int',
'index' => 'not_analyzed',
'include_in_all' => FALSE
),
// list of groups for which only admins have access to the artefact
'admin' => array (
'type' => 'int',
'index' => 'not_analyzed',
'include_in_all' => FALSE
),
// list of groups for which only members have access to the artefact
'member' => array (
'type' => 'int',
'index' => 'not_analyzed',
'include_in_all' => FALSE
),
// list of groups for which only tutors have access to the artefact
'tutor' => array (
'type' => 'int',
'index' => 'not_analyzed',
'include_in_all' => FALSE
),
*/
),
'type' => 'object',
'include_in_all' => FALSE,
'properties' => array (
'all' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'admin' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'member' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
),
'tutor' => array (
'type' => 'integer',
'index' => 'not_analyzed',
'copy_to' => 'group',
'include_in_all' => false
)
)
),
'group' => array (
'type' => 'integer'
),
......
How to upgrade your Mahara elasticsearch server from 1.7.x to 5.x
-----------------------------------------------------------------
Now that Mahara uses the 5.x version of the elasticsearch server you
will need to upgrade. The following steps should get you to where you
need to be. For more information see:
https://www.elastic.co/guide/en/elasticsearch/reference/current/reindex-upgrade.html
We normally can not jump from 1.7.x to 5.x directly as our current
index will not work. However, seeing as we can rebuild an index via
Mahara administration interface we can drop the index and rebuilding it later.
This will mean the search will be temporarily unavailable while it
reindexes.
Steps for doing the upgrade on a linux based server
Step 1:
-------
A good thing to do first up is do a search on your site and make note of the
results so that you can compare this with the same search done after upgrade.
Then we need to fetch the package we will be installing so create a dir to hold
the deb packages, eg
cd ~/ && mkdir elasticsearch_packages && cd elasticsearch_packages
Then fetch the elasticsearch version you want, eg
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.5.1.deb
Step 2:
-------
Check that the elasticserver is currently running, normally this is on
port 9200
curl -XGET 'http://localhost:9200'
It should return some info including version number, eg 1.7.3, then
also check the health of the cluster
curl -XGET 'localhost:9200/_cat/health?pretty'
Make sure status is 'green' before we begin. If it is not sort out any
problems so it returns 'green'.
Step 3:
-------
Install the downloaded version .deb package, eg
sudo dpkg -i elasticsearch-5.5.1.deb
It should output something like:
Unpacking elasticsearch (5.5.1) over (1.7.3) ...
Setting up elasticsearch (5.5.1) ...
Step 4:
-------
You elasticsearch will still be running on the old version so now is
the time to delete the current index you are using, if you don't know the
index you are using you can list indices via
curl -XGET 'localhost:9200/_cat/indices?pretty'
To delete the index called 'mahara' for example you go
curl -XDELETE 'localhost:9200/mahara'
Once that is done check health
curl -XGET 'localhost:9200/_cat/health?pretty'
Make sure status is still 'green'
Step 5:
-------
We need to stop / start the service to get it using the new 5.5.1
version
sudo -i service elasticsearch stop
sudo -i service elasticsearch start
Wait a few seconds for it to start up then
curl -XGET 'http://localhost:9200'
It should return some info including version number 5.5.1
If the elasticsearch server does not come up then check the
elasticsearch logs to see what went wrong
tail -n100 /var/log/elasticsearch/elasticsearch.log
Then check status of cluster again:
curl -XGET 'localhost:9200/_cluster/health?pretty'
Make sure status is still 'green'. Once the elasticsearch server is up
again we will need to reindex the site.
Step 6:
-------
Go to Mahara site in browser and go to Administration -> Extensions.
Under "Plugin type: search" configure the elasticsearch plugin.
Find and click 'Reset' in index reset section and that will begin the
re-indexing process.
Once re-indexing has finished do a search to make sure things are all working
again as expected. Do a search on the site and compare to one you did
in step 1 - the results should be the same
Troubleshooting:
----------------
If you find the status of the cluster is 'yellow' it probably means the
cluster is unbalanced so we check to see if there are any unassigned shards.
curl -XGET 'localhost:9200/_cat/shards?pretty'
If you are using only 1 node and there are unassigned replica shards
for an index we can turn them off, eg for 'mahara' index
curl -XPUT 'localhost:9200/mahara/_settings' -H 'Content-Type: application/json' -d' { "number_of_replicas": 0 }'
If you are finding that elasticsearch is not starting up again and you see
"failed to read local state, exiting..." in elasticsearch.log you might need to delete the
elasticsearch data directory.
In Ubuntu/Debian this can be found at /var/lib/elasticsearch/
\ No newline at end of file
......@@ -13,5 +13,5 @@ defined('INTERNAL') || die();
$config = new stdClass();
$config->name = 'elasticsearch';
$config->version = 2015100800;
$config->release = '1.0.4';
$config->version = 2017080300;
$config->release = '2.0.0';
......@@ -56,7 +56,11 @@
</div>
{/if}
<div class="detail">
{$record->description|str_shorten_html:100:true|safe}
{if $record->highlight}
{$record->highlight|safe}
{else}
{$record->description|str_shorten_html:140:true|safe}
{/if}
</div>
<!-- TAGS -->
{if $record->tags|count gt 0}
......
......@@ -20,7 +20,13 @@
{if $record->createdbyname}
<div class="createdby">{str tag=createdby section=search.elasticsearch arg1='<a href="`$record->createdby|profile_url`">`$record->createdbyname|safe`</a>'}</div>
{/if}
<div class="content-text">{$record->description|str_shorten_html:100:true|safe}</div>
<div class="content-text">