Commit ebd928b1 authored by Robert Lyon's avatar Robert Lyon

Bug 1692385: Adjusting the event_log table

Add 5 new new columns to event_log table make searching over the
data easier and also record the id of any parent/related info as well

Eg, if one deletes an image block from a page we now record:
id - the event id
resourceid - the block id
resourcetype - the blocktype, eg image
parentresourceid - the id of the parent, eg view id
parentresourcetype - the type of parent, eg 'view'

Because we would not be able to find related view info from just the
block id anymore.

Also added the ability to index and search over event information in
elasticsearch search type event_log

Change-Id: I280c1c75c35a6c58f42d8acb36cf5c24c70b902d
Signed-off-by: Robert Lyon's avatarRobert Lyon <robertl@catalyst.net.nz>
parent 408dcdfb
......@@ -138,7 +138,7 @@ class PluginArtefactBlog extends PluginArtefact {
$name = display_name($user, null, true);
$blog = new ArtefactTypeBlog(0, (object) array(
'title' => get_string('defaultblogtitle', 'artefact.blog', $name),
'owner' => $user['id'],
'owner' => is_object($user) ? $user->id : $user['id'],
));
$blog->commit();
}
......
......@@ -146,6 +146,7 @@ class PluginArtefactFile extends PluginArtefact {
}
public static function newuser($event, $user) {
$user = is_object($user) ? (array)$user : $user;
if (empty($user['quota'])) {
update_record('usr', array('quota' => get_config_plugin('artefact', 'file', 'defaultquota')), array('id' => $user['id']));
}
......
......@@ -711,7 +711,12 @@ abstract class ArtefactType implements IArtefactType {
$this->log('deleted');
}
handle_event('deleteartefact', $this);
$ignorefields = array(
'dirty', 'deleted', 'mtime', 'atime',
'tags', 'allowcomments', 'approvecomments', 'path'
);
handle_event('deleteartefact', $this, $ignorefields);
// Set flags.
$this->dirty = false;
......@@ -772,8 +777,8 @@ abstract class ArtefactType implements IArtefactType {
}
call_static_method($classname, 'bulk_delete', $ids);
}
handle_event('deleteartefacts', $artefactids);
$logdata = array_merge($containers, $leaves);
handle_event('deleteartefacts', $logdata);
db_commit();
}
......
......@@ -865,6 +865,10 @@ class BlockInstance {
return '';
}
public function to_stdclass() {
return (object)get_object_vars($this);
}
/**
* Builds the HTML for the block, inserting the blocktype content at the
* appropriate place
......@@ -1397,9 +1401,15 @@ class BlockInstance {
$this->dirty = false;
return;
}
$ignorefields = array('order', 'dirty',
'ignoreconfigdata' => array('retractable',
'removeoncancel',
'sure',
'retractedonload'
)
);
//Propagate the deletion of the block
handle_event('deleteblockinstance', $this);
handle_event('deleteblockinstance', $this, $ignorefields);
db_begin();
safe_require('blocktype', $this->get('blocktype'), 'lib.php', 'require_once', true);
......
......@@ -156,7 +156,7 @@ function activity_get_users($activitytype, $userids=null, $userobjs=null, $admin
* id
*/
function activity_set_defaults($eventdata) {
$user_id = $eventdata['id'];
$user_id = is_object($eventdata) ? $eventdata->id : $eventdata['id'];
$activitytypes = get_records_array('activity_type', 'admin', 0);
foreach ($activitytypes as $type) {
......
......@@ -99,14 +99,24 @@ class Collection {
public static function save($data) {
if (array_key_exists('id', $data)) {
$id = $data['id'];
$state = 'updatecollection';
}
else {
$id = 0;
$state = 'createcollection';
}
$collection = new Collection($id, $data);
$collection->set('mtime', time());
$collection->commit();
$views = $collection->get('views');
$viewids = array();
foreach ($views as $view) {
$viewids[] = $view->view;
}
$eventdata = array('id' => $collection->get('id'),
'name' => $collection->get('name'),
'viewids' => $viewids);
handle_event($state, $eventdata);
return $collection; // return newly created Collections id
}
......@@ -147,7 +157,10 @@ class Collection {
if ($viewids) {
delete_records_select('view_access', 'view IN (' . join(',', $viewids) . ') AND token IS NOT NULL');
}
$data = array('id' => $this->id,
'name' => $this->name,
'viewids' => $viewids);
handle_event('deletecollection', $data);
db_commit();
}
......@@ -1136,6 +1149,11 @@ class Collection {
View::_db_release($viewids, $this->owner, $this->submittedgroup);
db_commit();
handle_event('releasesubmission', array('releaseuser' => $releaseuser,
'id' => $this->get('id'),
'groupname' => $this->submittedgroup,
'eventfor' => 'collection'));
// We don't send out notifications about the release of remote-submitted Views & Collections
// (though I'm not sure why)
// if the method is called in an upgrade and we dont have a release user
......@@ -1263,6 +1281,11 @@ class Collection {
$this->commit();
db_commit();
handle_event('addsubmission', array('id' => $this->id,
'eventfor' => 'collection',
'name' => $this->name,
'group' => ($group) ? $group->id : null,
'groupname' => ($group) ? $group->name : null));
if ($group) {
activity_occurred(
'groupmessage',
......@@ -1330,7 +1353,10 @@ class Collection {
$todb->ctime = $access->ctime;
insert_record('view_access', $todb);
}
handle_event('updateviewaccess', array('id' => $this->id,
'eventfor' => 'collection',
'viewids' => $viewids,
'rules' => array($todb)));
return $access;
}
......
......@@ -310,11 +310,16 @@
</TABLE>
<TABLE NAME="event_log">
<FIELDS>
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true" />
<FIELD NAME="usr" TYPE="int" LENGTH="10" NOTNULL="true" />
<FIELD NAME="realusr" TYPE="int" LENGTH="10" NOTNULL="true" />
<FIELD NAME="time" TYPE="datetime" NOTNULL="true" />
<FIELD NAME="event" TYPE="char" LENGTH="255" NOTNULL="false" />
<FIELD NAME="data" TYPE="text" LENGTH="big" NOTNULL="false" />
<FIELD NAME="resourcetype" TYPE="char" LENGTH="255" NOTNULL="false" />
<FIELD NAME="resourceid" TYPE="int" LENGTH="10" NOTNULL="false" />
<FIELD NAME="parentresourcetype" TYPE="char" LENGTH="255" NOTNULL="false" />
<FIELD NAME="parentresourceid" TYPE="int" LENGTH="10" NOTNULL="false" />
</FIELDS>
<KEYS>
<KEY NAME="usrfk" TYPE="foreign" FIELDS="usr" REFTABLE="usr" REFFIELDS="id" />
......
......@@ -5018,11 +5018,6 @@ function xmldb_core_upgrade($oldversion=0) {
}
}
if ($oldversion < 2017052900) {
log_debug('Clear menu cache for removal of menu items');
clear_menu_cache();
}
if ($oldversion < 2017061200) {
log_debug('Add new logoxs column in institution table for small logos');
$table = new XMLDBTable('institution');
......@@ -5097,5 +5092,177 @@ function xmldb_core_upgrade($oldversion=0) {
}
}
if ($oldversion < 2017090800) {
log_debug('Clear menu cache for removal of menu items');
clear_menu_cache();
}
if ($oldversion < 2017090800) {
log_debug('Add new fields to "event_log" table');
// Instead of recording event resource id information in the 'data' json blob
// we will add it to it's own columns for easier searching / faster retrieval
// We will record if necessary the resourcetype/resourceid (and parenttype/parentid if necessary)
// And for elasticsearch we will need to add an id column to the table and change 'time' to 'ctime'.
$table = new XMLDBTable('event_log');
$field = new XMLDBField('id');
if (!field_exists($table, $field)) {
log_debug('Making a temp copy and adding id column');
execute_sql('CREATE TEMPORARY TABLE {temp_event_log} AS SELECT DISTINCT * FROM {event_log}', array());
if (is_mysql()) {
// We've disabled the db_start() method for our MySQL driver, but since we're truncating event_log,
// we really should start a transaction manually at least.
execute_sql('START TRANSACTION');
}
execute_sql('TRUNCATE {event_log}', array());
if (is_mysql()) {
// MySQL requires the auto-increment column to be a primary key right away.
execute_sql('ALTER TABLE {event_log} ADD id BIGINT(10) NOT NULL auto_increment PRIMARY KEY FIRST');
}
else {
$field->setAttributes(XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, XMLDB_SEQUENCE);
add_field($table, $field);
}
// Add 'ctime' field and drop 'time' field
$field = new XMLDBField('ctime');
$field->setAttributes(XMLDB_TYPE_DATETIME, null, null, XMLDB_NOTNULL);
add_field($table, $field);
$field = new XMLDBField('time');
drop_field($table, $field);
log_debug('Adding back in the event_log information');
// We will do in chuncks for large sites.
$count = 0;
$x = 0;
$limit = 1000;
$total = count_records('temp_event_log');
for ($i = 0; $i <= $total; $i += $limit) {
if (is_postgres()) {
$limitsql = ' OFFSET ' . $i . ' LIMIT ' . $limit;
}
else {
$limitsql = ' LIMIT ' . $i . ',' . $limit;
}
execute_sql('INSERT INTO {event_log} (usr, realusr, ctime, event, data) SELECT usr, realusr, time, event, data FROM {temp_event_log}' . $limitsql, array());
$count += $limit;
if (($count % ($limit *10)) == 0 || $count >= $total) {
if ($count > $total) {
$count = $total;
}
log_debug("$count/$total");
set_time_limit(30);
}
set_time_limit(30);
}
if (is_mysql()) {
execute_sql('COMMIT');
}
execute_sql('DROP TABLE {temp_event_log}', array());
if (!is_mysql()) {
log_debug('Adding primary key index to event_log.id column');
$key = new XMLDBKey('primary');
$key->setAttributes(XMLDB_KEY_PRIMARY, array('id'));
add_key($table, $key);
}
}
$field = new XMLDBField('resourceid');
$field->setAttributes(XMLDB_TYPE_INTEGER, 10);
add_field($table, $field);
$field = new XMLDBField('resourcetype');
$field->setAttributes(XMLDB_TYPE_CHAR, 255);
add_field($table, $field);
$field = new XMLDBField('parentresourceid');
$field->setAttributes(XMLDB_TYPE_INTEGER, 10);
add_field($table, $field);
$field = new XMLDBField('parentresourcetype');
$field->setAttributes(XMLDB_TYPE_CHAR, 255);
add_field($table, $field);
log_debug('Adjust existing "event_log" data to fit new table structure');
// As there can be very many rows we need to do the adjusting in chuncks
$count = 0;
$limit = 10000;
$chunk = 5000;
$total = count_records_select('event_log', 'data != ?', array('{}'));
if ($total > 0) {
for ($i = 0; $i <= $total; $i += $chunk) {
$results = get_records_sql_array("SELECT event, data FROM {event_log}", array(), $count, $chunk);
foreach ($results as $result) {
$data = json_decode($result->data);
$where = clone $result;
switch ($result->event) {
case 'saveview':
case 'deleteview':
$result->resourceid = $data->id;
$result->resourcetype = 'view';
break;
case 'userjoinsgroup':
$result->resourceid = $data->group;
$result->resourcetype = 'group';
break;
case 'creategroup':
$result->resourceid = $data->id;
$result->resourcetype = 'group';
break;
case 'saveartefact':
case 'deleteartefact':
case 'deleteartefacts':
$result->resourceid = $data->id;
$result->resourcetype = $data->artefacttype;
break;
case 'blockinstancecommit':
case 'deleteblockinstance':
$result->resourceid = $data->id;
$result->resourcetype = $data->blocktype;
break;
case 'addfriend':
case 'removefriend':
$result->resourceid = $data->friend;
$result->resourcetype = 'friend';
break;
case 'addfriendrequest':
$result->resourceid = $data->owner;
$result->resourcetype = 'friend';
break;
case 'removefriendrequest':
$result->resourceid = $data->requester;
$result->resourcetype = 'friend';
break;
}
update_record('event_log', $result, $where);
}
$count += $chunk;
if (($count % $limit) == 0 || $count >= $total) {
if ($count > $total) {
$count = $total;
}
log_debug("$count/$total");
set_time_limit(30);
}
}
}
log_debug('Add new logging events');
$newevents = array('createview',
'createcollection',
'updatecollection',
'deletecollection',
'addsubmission',
'releasesubmission',
'updateviewaccess');
foreach ($newevents as $newevent) {
$event = (object)array(
'name' => $newevent,
);
ensure_record_exists('event_type', $event, $event);
}
}
return $status;
}
......@@ -539,12 +539,16 @@ function group_create($data) {
}
}
}
insert_record('view_access', (object) array(
$newaccess = (object) array(
'view' => $homepage->get('id'),
'accesstype' => $data['public'] ? 'public' : 'loggedin',
'ctime' => db_format_timestamp(time()),
));
);
insert_record('view_access', $newaccess);
handle_event('updateviewaccess', array('id' => $id,
'eventfor' => 'group',
'viewids' => $homepage->get('id'),
'rules' => array($newaccess)));
handle_event('creategroup', $data);
db_commit();
......@@ -728,20 +732,26 @@ function group_update($new, $create=false) {
if ($old->public != $new->public) {
if ($old->public && !$new->public) {
delete_records('view_access', 'view', $homepageid, 'accesstype', 'public');
insert_record('view_access', (object) array(
$newaccess = (object) array(
'view' => $homepageid,
'accesstype' => 'loggedin',
'ctime' => db_format_timestamp(time()),
));
);
insert_record('view_access', $newaccess);
}
else if (!$old->public && $new->public) {
delete_records('view_access', 'view', $homepageid, 'accesstype', 'loggedin');
insert_record('view_access', (object) array(
$newaccess = (object) array(
'view' => $homepageid,
'accesstype' => 'public',
'ctime' => db_format_timestamp(time()),
));
);
insert_record('view_access', $newaccess);
}
handle_event('updateviewaccess', array('id' => $new->id,
'eventfor' => 'group',
'viewids' => $homepageid,
'rules' => array($newaccess)));
}
// When the create/edit permissions change, update permissions on journal and posts
......@@ -971,6 +981,9 @@ function group_add_user($groupid, $userid, $role=null, $method='internal') {
insert_record('group_member', $gm);
delete_records('group_member_request', 'group', $groupid, 'member', $userid);
delete_records('group_member_invite', 'group', $groupid, 'member', $userid);
$gm->id = $gm->group;
$gm->eventfor = 'group';
handle_event('userjoinsgroup', $gm);
db_commit();
global $USER;
......
......@@ -1851,7 +1851,7 @@ function blocktype_artefactplugin($blocktype) {
/**
* Fires an event which can be handled by different parts of the system
*/
function handle_event($event, $data) {
function handle_event($event, $data, $ignorefields = array()) {
global $USER;
static $event_types = array(), $coreevents_cache = array(), $eventsubs_cache = array();
......@@ -1865,25 +1865,87 @@ function handle_event($event, $data) {
throw new SystemException("Invalid event");
}
if ($data instanceof ArtefactType) {
// leave $data alone, but convert for the event log
$logdata = $data->to_stdclass();
// leave $data alone, but convert for the event log
if (is_object($data)) {
$logdata = clone $data;
}
else if ($data instanceof BlockInstance) {
// leave $data alone, but convert for the event log
$logdata = array(
'id' => $data->get('id'),
'blocktype' => $data->get('blocktype'),
);
else {
if (is_numeric($data)) {
$logdata = $data = array('id' => $data);
}
else {
$logdata = $data;
}
$data = (array)$data;
}
else if (is_object($data)) {
if (isset($data->password)) {
unset($data->password);
$refid = $reftype = $parentrefid = $parentreftype = null;
// Need to set dirty to false for all the classes with destructors
if ($logdata instanceof View) {
$logdata->set('dirty', false);
// Move the id / view off to dedicated columns for easier searching
$logdata = $logdata->to_stdclass();
$refid = $logdata->id;
unset($logdata->id);
$reftype = 'view';
}
else if ($logdata instanceof ArtefactType) {
$logdata->set('dirty', false);
// Move the id / atefacttype off to dedicated columns for easier searching
$logdata = $logdata->to_stdclass();
$refid = $logdata->id;
unset($logdata->id);
$reftype = $logdata->artefacttype;
unset($logdata->artefacttype);
}
else if ($logdata instanceof BlockInstance) {
$logdata->set('dirty', false);
// Remove data from configdata that we don't need to log
$configdata = $logdata->get('configdata');
if (isset($ignorefields['ignoreconfigdata'])) {
$ignore = $ignorefields['ignoreconfigdata'];
if (!empty($ignore) && is_array($ignore)) {
foreach ($ignore as $item) {
unset($configdata[$item]);
}
}
unset($ignorefields['ignoreconfigdata']);
}
$logdata = $logdata->to_stdclass();
$logdata->configdata = $configdata;
// Move the id / blocktype and parent id / view off to dedicated columns for easier searching
$refid = $logdata->id;
unset($logdata->id);
$reftype = $logdata->blocktype;
unset($logdata->blocktype);
$parentrefid = $logdata->view;
unset($logdata->view);
$parentreftype = 'view';
}
else if (is_object($logdata)) {
// Try to set id / type from stdclass object to dedicated column if 'eventfor' indicated
// eg. event = creategroup would have eventfor = group
$logdata = (object)get_object_vars($logdata);
if (isset($logdata->id) && isset($logdata->eventfor)) {
$refid = $logdata->id;
$reftype = $logdata->eventfor;
unset($logdata->eventfor);
}
$data = (array)$data;
}
else if (is_numeric($data)) {
$data = array('id' => $data);
else {
$refid = !empty($logdata['id']) ? $logdata['id'] : null;
$reftype = !empty($logdata['eventfor']) ? $logdata['eventfor'] : null;
unset($logdata['eventfor']);
}
// Then remove any unwanted items
if (is_object($logdata)) {
foreach ($logdata as $key => $field) {
if (in_array($key, $ignorefields) || empty($field)) {
unset($logdata->{$key});
}
}
$logdata = (array)$logdata;
}
$parentuser = $USER->get('parentuser');
......@@ -1894,8 +1956,12 @@ function handle_event($event, $data) {
'usr' => $USER->get('id'),
'realusr' => $parentuser ? $parentuser->id : $USER->get('id'),
'event' => $event,
'data' => json_encode(isset($logdata) ? $logdata : $data),
'time' => db_format_timestamp(time()),
'data' => json_encode($logdata),
'ctime' => db_format_timestamp(time()),
'resourceid' => $refid,
'resourcetype' => $reftype,
'parentresourceid' => $parentrefid,
'parentresourcetype' => $parentreftype,
);
insert_record('event_log', $logentry);
}
......@@ -4284,7 +4350,7 @@ function cron_event_log_expire() {
if ($expiry = get_config('eventlogexpiry')) {
delete_records_select(
'event_log',
'time < CURRENT_DATE - INTERVAL ' .
'ctime < CURRENT_DATE - INTERVAL ' .
(is_postgres() ? "'" . $expiry . " seconds'" : $expiry . ' SECOND')
);
......
......@@ -806,7 +806,7 @@ function core_install_lastcoredata_defaults() {
set_profile_field($user->id, 'email', $user->email);
set_profile_field($user->id, 'firstname', $user->firstname);
set_profile_field($user->id, 'lastname', $user->lastname);
handle_event('createuser', $user);
handle_event('createuser', $user, array('password'));
activity_add_admin_defaults(array($user->id));
db_commit();
......@@ -888,6 +888,13 @@ function core_install_firstcoredata_defaults() {
'creategroup',
'loginas',
'clearcaches',
'createview',
'createcollection',
'updatecollection',
'deletecollection',
'addsubmission',
'releasesubmission',
'updateviewaccess'
);
foreach ($eventtypes as $et) {
......
......@@ -2509,7 +2509,7 @@ function create_user($user, $profile=array(), $institution=null, $remoteauth=nul
reset_password($user, false, $quickhash);
$createuser = clone $user;
handle_event('createuser', $createuser);
handle_event('createuser', $createuser, array('password'));
db_commit();
return $user->id;
}
......@@ -2617,7 +2617,7 @@ function update_user($user, $profile, $remotename=null, $accountprefs=array(), $
*/
function add_user_to_autoadd_groups($eventdata) {
require_once('group.php');
$userid = $eventdata['id'];
$userid = is_object($eventdata) ? $eventdata->id : $eventdata['id'];
if ($autoaddgroups = get_column('group', 'id', 'usersautoadded', true)) {
foreach ($autoaddgroups as $groupid) {
if (!group_user_access($groupid, $userid)) {
......
......@@ -16,7 +16,7 @@ $config = new stdClass();
// See https://wiki.mahara.org/wiki/Developer_Area/Version_Numbering_Policy
// For upgrades on stable branches, increment the version by one. On master, use the date.
$config->version = 2017090400;
$config->version = 2017090800;
$config->series = '17.10';
$config->release = '17.10dev';
$config->minupgradefrom = 2012080604;
......
......@@ -515,10 +515,13 @@ class View {
$obj->usr = $template->get('owner');
$obj->group = $template->get('group');
insert_record('view_access', $obj);
handle_event('updateviewaccess', array('id' => $view->get('id'),
'eventfor' => 'view',
'viewids' => $view->get('id'),
'rules' => array($obj)));
}
db_commit();
return array(
$view,
$template,
......@@ -609,6 +612,7 @@ class View {
$view = new View(0, $data);
$view->commit();
if (isset($viewdata['group']) &&
(empty($viewdata['type']) || (!empty($viewdata['type']) && $viewdata['type'] != 'grouphomepage'))
) {
......@@ -619,12 +623,16 @@ class View {
$beforeusers[$userid] = get_record('usr', 'id', $userid);
// By default, group views should be visible to the group
insert_record('view_access', (object) array(
$newaccess = (object) array(
'view' => $view->get('id'),
'group' => $viewdata['group'],
'ctime' => db_format_timestamp(time()),
));
);
insert_record('view_access', $newaccess);
handle_event('updateviewaccess', array('id' => $viewdata['group'],
'eventfor' => 'group',
'viewids' => $view->get('id'),
'rules' => array($newaccess)));
// Notify group members
$accessdata = new StdClass;
$accessdata->view = $view->get('id');
......@@ -759,9 +767,11 @@ class View {
throw new SystemException(get_string('onlonlyyoneprofileviewallowed', 'error'));
}
$this->id = insert_record('view', $fordb, 'id', true);
handle_event('createview', $this->id);
}
else {
update_record('view', $fordb, 'id');
handle_event('saveview', $this->id);
}
if (isset($this->tags)) {
......@@ -867,7 +877,9 @@ class View {
delete_records('view_autocreate_grouptype', 'view', $this->id);
delete_records('view_tag','view',$this->id);
delete_records('view_visit','view',$this->id);
$eventdata = array('id' => $this->id);
if ($collection = $this->get_collection()) {