Skip to content

GitLab

Commit 064531f4 authored by Richard Mansfield's avatar Richard Mansfield
Browse files

Faster deletion of folders with lots of artefacts inside them 


Signed-off-by: default avatarRichard Mansfield <richardm@catalyst.net.nz>
parent f176da93
Hide whitespace changes
Inline Side-by-side
htdocs/artefact/file/lib.php
View file @ 064531f4
......@@ -961,6 +961,64 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
parent::delete();
}
public static function bulk_delete($artefactids) {
global $USER;
if (empty($artefactids)) {
return;
}
$idstr = join(',', $artefactids);
db_begin();
// Get the size of all the files we're about to delete that belong to
// the user.
$totalsize = get_field_sql('
SELECT SUM(size)
FROM {artefact_file_files} f JOIN {artefact} a ON f.artefact = a.id
WHERE a.owner = ? AND f.artefact IN (' . $idstr . ')',
array($USER->get('id'))
);
// Get all fileids so that we can delete the files on disk
$fileids = get_records_select_assoc('artefact_file_files', 'artefact IN (' . $idstr . ')', array());
$fileidcounts = get_records_sql_assoc('
SELECT fileid, COUNT(fileid)
FROM {artefact_file_files}
WHERE artefact IN (' . $idstr . ')
GROUP BY fileid',
null
);
// The current rule is that file deletion should be logged in the artefact_log table
// only for group-owned files. To save time we will be slightly naughty here and
// log deletion for all these files if at least one is group-owned.
$log = (bool) count_records_select('artefact', 'id IN (' . $idstr . ') AND "group" IS NOT NULL');
set_field_select('view_feedback', 'attachment', null, 'attachment IN (' . $idstr . ')', array());
delete_records_select('artefact_attachment', 'attachment IN (' . $idstr . ')');
delete_records_select('artefact_file_files', 'artefact IN (' . $idstr . ')');
parent::bulk_delete($artefactids, $log);
foreach ($fileids as $r) {
// Delete the file on disk if there's only one artefact left pointing to it
if ($fileidcounts[$r->fileid]->count == 1) {
$file = get_config('dataroot') . self::get_file_directory($r->fileid) . '/' . $r->fileid;
if (is_file($file)) {
unlink($file);
}
}
$fileidcounts[$r->fileid]->count--;
}
if ($totalsize) {
$USER->quota_remove($totalsize);
$USER->commit();
}
db_commit();
}
public static function has_config() {
return true;
}
......@@ -1155,6 +1213,19 @@ class ArtefactTypeFolder extends ArtefactTypeFileBase {
}
public function delete() {
// ArtefactType::delete() deletes all the child artefacts one by one.
// If the folder contains a lot of artefacts, it's too slow to do this
// but for very small directories it seems to be slightly faster.
$descendants = artefact_get_descendants(array($this->id));
if (count($descendants) < 10) {
parent::delete();
}
else {
ArtefactType::delete_by_artefacttype($descendants);
}
}
public function folder_contents() {
return get_records_array('artefact', 'parent', $this->get('id'));
}
......@@ -1443,6 +1514,16 @@ class ArtefactTypeImage extends ArtefactTypeFile {
parent::delete();
}
public static function bulk_delete($artefactids) {
if (empty($artefactids)) {
return;
}
db_begin();
delete_records_select('artefact_file_image', 'artefact IN (' . join(',', $artefactids) . ')');
parent::bulk_delete($artefactids);
db_commit();
}
public function render_self($options) {
$result = parent::render_self($options);
$result['html'] = '<div class="fr filedata-icon" style="text-align: center;"><h4>' . get_string('Preview', 'artefact.file') . '</h4><a href="'
......
htdocs/artefact/lib.php
View file @ 064531f4
......@@ -429,7 +429,7 @@ abstract class ArtefactType {
}
}
artefact_watchlist_notification($this->id);
artefact_watchlist_notification(array($this->id));
handle_event('saveartefact', $this);
......@@ -482,32 +482,14 @@ abstract class ArtefactType {
}
}
artefact_watchlist_notification($this->id);
artefact_watchlist_notification(array($this->id));
// Delete any references to this artefact from non-artefact places.
delete_records_select('artefact_parent_cache', 'artefact = ? OR parent = ?', array($this->id, $this->id));
// Make sure that the artefact is removed from any view blockinstances that have it
if ($records = get_column('view_artefact', 'block', 'artefact', $this->id)) {
foreach ($records as $blockid) {
require_once(get_config('docroot') . 'blocktype/lib.php');
$bi = new BlockInstance($blockid);
$bi->delete_artefact($this->id);
}
}
delete_records('view_artefact', 'artefact', $this->id);
delete_records('artefact_feedback', 'artefact', $this->id);
delete_records('artefact_tag', 'artefact', $this->id);
delete_records('artefact_access_role', 'artefact', $this->id);
delete_records('artefact_access_usr', 'artefact', $this->id);
self::_delete_dbrecords(array($this->id));
if ($this->can_be_logged()) {
$this->log('deleted');
}
// Delete the record itself.
delete_records('artefact', 'id', $this->id);
handle_event('deleteartefact', $this);
// Set flags.
......@@ -518,6 +500,117 @@ abstract class ArtefactType {
db_commit();
}
/**
* Does a bulk_delete on a list of artefacts, grouping artefacts of
* the same type.
*
* Currently only tested for folders and their contents.
*/
public static function delete_by_artefacttype($artefactids) {
if (empty($artefactids)) {
return;
}
db_begin();
artefact_watchlist_notification($artefactids);
$records = get_records_select_assoc(
'artefact',
'id IN (' . join(',', $artefactids) . ')',
null, 'artefacttype', 'id,parent,artefacttype,container'
);
$containers = array();
$leaves = array();
foreach ($records as $r) {
if ($r->container) {
$containers[$r->artefacttype][] = $r->id;
}
else {
$leaves[$r->artefacttype][] = $r->id;
}
}
// Delete non-containers grouped by artefacttype
foreach ($leaves as $artefacttype => $ids) {
$classname = generate_artefact_class_name($artefacttype);
call_static_method($classname, 'bulk_delete', $ids);
}
// Delete containers grouped by artefacttype
foreach ($containers as $artefacttype => $ids) {
$classname = generate_artefact_class_name($artefacttype);
call_static_method($classname, 'bulk_delete', $ids);
}
handle_event('deleteartefacts', $artefactids);
db_commit();
}
/**
* Faster delete for multiple artefacts.
*
* Should only be called on artefacts with no children, after
* additional data in other tables has already been deleted.
*/
public static function bulk_delete($artefactids, $log=false) {
db_begin();
self::_delete_dbrecords($artefactids);
// Logging must be triggered by the caller because it's
// slow to go through each artefact and ask it if it should
// be logged.
if ($log) {
global $USER;
$entry = (object) array(
'usr' => $USER->get('id'),
'time' => db_format_timestamp(time()),
'deleted' => 1,
);
foreach ($artefactids as $id) {
$entry->artefact = $id;
insert_record('artefact_log', $entry);
}
}
db_commit();
}
private static function _delete_dbrecords($artefactids) {
if (empty($artefactids)) {
return;
}
$idstr = '(' . join(',', $artefactids) . ')';
db_begin();
// Delete any references to these artefacts from non-artefact places.
delete_records_select('artefact_parent_cache', "artefact IN $idstr");
// Make sure that the artefacts are removed from any view blockinstances
if ($records = get_records_sql_array("
SELECT va.block, va.artefact, bi.configdata
FROM {view_artefact} va JOIN {block_instance} bi ON va.block = bi.id
WHERE va.artefact IN $idstr", array())) {
require_once(get_config('docroot') . 'blocktype/lib.php');
BlockInstance::bulk_delete_artefacts($records);
}
delete_records_select('view_artefact', "artefact IN $idstr");
delete_records_select('artefact_feedback', "artefact IN $idstr");
delete_records_select('artefact_tag', "artefact IN $idstr");
delete_records_select('artefact_access_role', "artefact IN $idstr");
delete_records_select('artefact_access_usr', "artefact IN $idstr");
delete_records_select('artefact', "id IN $idstr");
db_commit();
}
/**
* this function provides the way to link to viewing very deeply nested artefacts
* within a view
......@@ -1071,28 +1164,47 @@ function artefact_get_attachment_types() {
return $artefacttypes;
}
function artefact_get_parents_for_cache($artefactid, &$parentids=false) {
$current = $artefactid;
function artefact_get_parents_for_cache($artefactids, &$parentids=false) {
if (is_integer($artefactids)) {
$artefactids = array($artefactids);
}
$current = $artefactids;
if (empty($parentids)) { // first call
$parentids = array();
}
while (true) {
if (!$parent = get_record('artefact', 'id', $current)) {
if (!$parents = get_records_select_array('artefact', 'id IN (' . join(',',$current) . ')')) {
break;
}
// get any blog posts it may be attached to
if (in_array($parent->artefacttype, artefact_get_attachment_types())
&& $associated = ArtefactType::attached_id_list($parent->id)) {
foreach ($associated as $a) {
$parentids[$a] = 1;
artefact_get_parents_for_cache($a, $parentids);
// get any blog posts these artefacts may be attached to
$checkattachments = array();
foreach ($parents as $p) {
if (in_array($p->artefacttype, artefact_get_attachment_types())) {
$checkattachments[] = $p->id;
}
}
if (!empty($checkattachments)) {
if ($associated = get_records_select_assoc('artefact_attachment', 'attachment IN (' . join(',', $checkattachments) . ')')) {
$associated = array_keys($associated);
foreach ($associated as $a) {
$parentids[$a] = 1;
}
artefact_get_parents_for_cache($associated, $parentids);
}
}
// check parents
$current = array();
foreach ($parents as $p) {
if ($p->parent) {
$parentids[$p->parent] = 1;
$current[] = $p->parent;
}
}
if (!$parent->parent) {
if (empty($current)) {
break;
}
$parentids[$parent->parent] = 1;
$current = $parent->parent;
}
return $parentids;
}
......@@ -1168,9 +1280,9 @@ function artefact_instance_from_type($artefact_type, $user_id=null) {
throw new ArtefactNotFoundException("Artefact of type '${artefact_type}' doesn't exist");
}
function artefact_watchlist_notification($artefactid) {
function artefact_watchlist_notification($artefactids) {
// gets all the views containing this artefact or a parent of this artefact and creates a watchlist activity for each view
if ($views = get_column_sql('SELECT DISTINCT view FROM {view_artefact} WHERE artefact IN (' . implode(',', array_merge(array_keys(artefact_get_parents_for_cache($artefactid)), array($artefactid))) . ')')) {
if ($views = get_column_sql('SELECT DISTINCT view FROM {view_artefact} WHERE artefact IN (' . implode(',', array_merge(array_keys(artefact_get_parents_for_cache($artefactids)), $artefactids)) . ')')) {
require_once('activity.php');
foreach ($views as $view) {
activity_occurred('watchlist', (object)array('view' => $view));
......
htdocs/blocktype/lib.php
View file @ 064531f4
......@@ -888,8 +888,10 @@ class BlockInstance {
* value 'artefactid' or in the array 'artefactids'
* 2) The block must ALWAYS continue to work even when artefacts are
* removed from it
*
* Don't override this method without doing the right thing in bulk_delete_artefacts too.
*/
public function delete_artefact($artefact) {
final public function delete_artefact($artefact) {
$configdata = $this->get('configdata');
$changed = false;
......@@ -913,6 +915,42 @@ class BlockInstance {
}
}
/**
* Deletes artefacts from the blockinstances given in $records.
* $records should be an array of stdclass objects, each containing
* a blockid, an artefactid, and the block's configdata
*/
public static function bulk_delete_artefacts($records) {
if (empty($records)) {
return;
}
$blocklist = array();
foreach ($records as $record) {
if (isset($blocklist[$record->block])) {
$blocklist[$record->block]->artefacts[] = $record->artefact;
}
else {
$blocklist[$record->block] = (object) array(
'artefacts' => array($record->artefact),
'configdata' => unserialize($record->configdata),
);
}
}
foreach ($blocklist as $blockid => $blockdata) {
if (isset($blockdata->configdata['artefactid'])) {
if ($change = $blockdata->configdata['artefactid'] == $blockdata->artefacts[0]) {
$blockdata->configdata['artefactid'] = null;
}
}
else if (isset($blockdata->configdata['artefactids'])) {
$blockdata->configdata['artefactids'] = array_diff($blockdata->configdata['artefactids'], $blockdata->artefacts);
$change = true;
}
if ($change) {
set_field('block_instance', 'configdata', serialize($blockdata->configdata), 'id', $blockid);
}
}
}
/**
* Get an artefact instance, checking republish permissions
......
htdocs/lib/db/upgrade.php
View file @ 064531f4
......@@ -1210,6 +1210,13 @@ function xmldb_core_upgrade($oldversion=0) {
}
}
if ($oldversion < 2009092900) {
$event = (object)array(
'name' => 'deleteartefacts',
);
ensure_record_exists('event_type', $event, $event);
}
return $status;
}
......
htdocs/lib/upgrade.php
View file @ 064531f4
......@@ -699,6 +699,7 @@ function core_install_firstcoredata_defaults() {
'userjoinsgroup',
'saveartefact',
'deleteartefact',
'deleteartefacts',
'saveview',
'deleteview',
'blockinstancecommit',
......
htdocs/lib/version.php
View file @ 064531f4
......@@ -28,7 +28,7 @@
defined('INTERNAL') || die();
$config = new StdClass;
$config->version = 2009092100;
$config->version = 2009092900;
$config->release = '1.2.0beta4dev';
$config->minupgradefrom = 2008040200;
$config->minupgraderelease = '1.0.0 (release tag 1.0.0_RELEASE)';
......
htdocs/search/solr/lib.php
View file @ 064531f4
......@@ -45,20 +45,21 @@ class PluginSearchSolr extends PluginSearchInternal {
public static function get_event_subscriptions() {
$subscriptions = array(
(object)array('plugin' => 'solr', 'event' => 'createuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'updateuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'suspenduser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'unsuspenduser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'deleteuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'undeleteuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'expireuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'unexpireuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'deactivateuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'activateuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'saveartefact', 'callfunction' => 'event_saveartefact' ),
(object)array('plugin' => 'solr', 'event' => 'deleteartefact', 'callfunction' => 'event_deleteartefact' ),
(object)array('plugin' => 'solr', 'event' => 'saveview', 'callfunction' => 'event_saveview' ),
(object)array('plugin' => 'solr', 'event' => 'deleteview', 'callfunction' => 'event_deleteview' ),
(object)array('plugin' => 'solr', 'event' => 'createuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'updateuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'suspenduser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'unsuspenduser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'deleteuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'undeleteuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'expireuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'unexpireuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'deactivateuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'activateuser', 'callfunction' => 'event_reindex_user' ),
(object)array('plugin' => 'solr', 'event' => 'saveartefact', 'callfunction' => 'event_saveartefact' ),
(object)array('plugin' => 'solr', 'event' => 'deleteartefact', 'callfunction' => 'event_deleteartefact' ),
(object)array('plugin' => 'solr', 'event' => 'deleteartefacts', 'callfunction' => 'event_deleteartefacts'),
(object)array('plugin' => 'solr', 'event' => 'saveview', 'callfunction' => 'event_saveview' ),
(object)array('plugin' => 'solr', 'event' => 'deleteview', 'callfunction' => 'event_deleteview' ),
);
return $subscriptions;
......@@ -85,6 +86,16 @@ class PluginSearchSolr extends PluginSearchInternal {
self::delete_byidtype($artefact->get('id'), 'artefact');
self::commit();
}
public static function event_deleteartefacts($event, $artefactids) {
if (!self::config_is_sane()) {
return;
}
// Should send these deletes in one go
foreach ($artefactids as $id) {
self::delete_byidtype($id, 'artefact');
}
self::commit();
}
public static function event_saveview($event, $view) {
if (!self::config_is_sane()) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment