Commit 28599c24 authored by Richard Mansfield's avatar Richard Mansfield
Browse files

Read artefact table in chunks when rebuilding parent cache (bug #820727)

The cron job to rebuild the artefact_parent_cache table uses all the
available memory when the artefact table gets big, so it needs to read
it in in smaller chunks.

This version of the function ensures that all artefacts with the same
owner are read in at the same time, which means all artefact parent
relationships are available.

See http://mahara.org/interaction/forum/topic.php?id=2467 and
http://mahara.org/interaction/forum/topic.php?id=2645

 for more
details.

Change-Id: I459a2d4a9637716809476c5547bdf1ab7e81f0d4
Signed-off-by: default avatarRichard Mansfield <richard.mansfield@catalyst.net.nz>
parent 3a83ed6e
......@@ -1244,52 +1244,88 @@ function rebuild_artefact_parent_cache_dirty() {
}
function rebuild_artefact_parent_cache_complete() {
// The artefact table may get too big to read in all at once. Because
// all artefacts' parents have the same owner (or group, institution) as
// the child artefact, we can read the table in chunks as long as all
// artefacts with the same owner are processed in one go.
// Attachment relationships also get records in artefact_parent_cache,
// but these are fetched separately in a per-artefact query inside
// rebuild_artefact_parent_cache_partial, and there is no assumption
// that attachments have the same owner as the artefact they're attached
// to.
db_begin();
delete_records('artefact_parent_cache');
$artefacts = get_records_sql_assoc('
SELECT id, parent, COUNT(aa.artefact) AS attached
FROM {artefact} a LEFT JOIN {artefact_attachment} aa ON a.id = aa.attachment
GROUP BY id, parent
HAVING parent IS NOT NULL OR COUNT(aa.artefact) > 0',
array()
);
foreach (array('owner', 'group', 'institution') as $f) {
$select = '
SELECT id, parent, COUNT(aa.artefact) AS attached
FROM {artefact} a LEFT JOIN {artefact_attachment} aa ON a.id = aa.attachment';
$where = "
WHERE a.$f IS NOT NULL";
$groupby = "
GROUP BY id, parent
HAVING parent IS NOT NULL OR COUNT(aa.artefact) > 0";
if ($f == 'group' || $f == 'institution') {
$sql = $select . $where . $groupby;
if ($artefacts = get_records_sql_assoc($sql, array())) {
rebuild_artefact_parent_cache_partial($artefacts);
}
continue;
}
if ($artefacts) {
// Individual user artefacts
// Process artefacts in chunks with $userlimit owners
$userlimit = 5000;
$maxuser = get_field('artefact', 'MAX(owner)');
$sql = $select . $where . ' AND owner >= ? AND owner < ?' . $groupby;
foreach ($artefacts as &$artefact) {
for ($i = 0; $i <= $maxuser; $i += $userlimit) {
if ($artefacts = get_records_sql_assoc($sql, array($i, $i + $userlimit))) {
rebuild_artefact_parent_cache_partial($artefacts);
}
}
}
// Nothing that can be a parent can be an attachment, so it's good
// enough to first get everything this artefact is attached to, and
// then find all its ancestors and the ancestors of everything it's
// attached to.
db_commit();
}
$ancestors = array();
if ($artefact->attached) {
$ancestors = get_column('artefact_attachment', 'artefact', 'attachment', $artefact->id);
}
function rebuild_artefact_parent_cache_partial($artefacts) {
$tocheck = $ancestors;
$tocheck[] = $artefact->id;
foreach ($artefacts as &$artefact) {
foreach ($tocheck as $id) {
$p = isset($artefacts[$id]) ? $artefacts[$id]->parent : null;
while (!empty($p)) {
$ancestors[] = $p;
$p = isset($artefacts[$p]) ? $artefacts[$p]->parent : null;
}
}
// Nothing that can be a parent can be an attachment, so it's good
// enough to first get everything this artefact is attached to, and
// then find all its ancestors and the ancestors of everything it's
// attached to.
foreach (array_unique($ancestors) as $p) {
insert_record('artefact_parent_cache', (object) array(
'artefact' => $artefact->id,
'parent' => $p,
'dirty' => 0,
));
$ancestors = array();
if ($artefact->attached) {
$ancestors = get_column('artefact_attachment', 'artefact', 'attachment', $artefact->id);
}
$tocheck = $ancestors;
$tocheck[] = $artefact->id;
foreach ($tocheck as $id) {
$p = isset($artefacts[$id]) ? $artefacts[$id]->parent : null;
while (!empty($p)) {
$ancestors[] = $p;
$p = isset($artefacts[$p]) ? $artefacts[$p]->parent : null;
}
}
foreach (array_unique($ancestors) as $p) {
insert_record('artefact_parent_cache', (object) array(
'artefact' => $artefact->id,
'parent' => $p,
'dirty' => 0,
));
}
}
db_commit();
}
function artefact_get_attachment_types() {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment