Commit 042f0da9 authored by Richard Mansfield's avatar Richard Mansfield
Browse files

Limit parent cache artefacts processed at one time (bug #820727)

In commit 28599c24

, the
rebuild_artefact_parent_cache_complete cron job was rewritten to
process artefacts in groups of 5000 owners.  This limit is too high;
I've seen the cron script run out of memory when trying to process
~50000 artefacts in memory at once.

This change makes the memory usage more predictable by trying to limit
the number of artefacts pulled into memory rather than the number of
owners, which is often not a good approximation.

Change-Id: If63dec213e9794e86f37461166eb0530a407e086
Signed-off-by: default avatarRichard Mansfield <richard.mansfield@catalyst.net.nz>
parent bcdb22cf
......@@ -1281,16 +1281,35 @@ function rebuild_artefact_parent_cache_complete() {
continue;
}
// Individual user artefacts
// Process artefacts in chunks with $userlimit owners
$userlimit = 5000;
// There are usually a lot of individual user artefacts, so these are processed in
// chunks. Ideally we would like to do < $artefactlimit artefacts at a time, but
// it's more important to make sure all artefacts with the same owner are processed
// together, so if one user has too many artefacts, this limit will be exceeded.
$artefactlimit = 25000;
$defaultuserlimit = 1000; // Try to do this many users in each run
$maxuser = get_field('artefact', 'MAX(owner)');
$sql = $select . $where . ' AND owner >= ? AND owner < ?' . $groupby;
for ($i = 0; $i <= $maxuser; $i += $userlimit) {
$countsql = 'SELECT COUNT(r.id) FROM (' . $sql . ') r';
$i = 0;
while ($i <= $maxuser) {
$userlimit = $defaultuserlimit;
$n = count_records_sql($countsql, array($i, $i + $userlimit));
// If we have too many artefacts, try fewer users.
while ($n > $artefactlimit) {
$userlimit = floor($userlimit / 2);
if ($userlimit < 2) {
break;
}
$n = count_records_sql($countsql, array($i, $i + $userlimit));
}
if ($artefacts = get_records_sql_assoc($sql, array($i, $i + $userlimit))) {
$lastuser = min($i + $userlimit, $maxuser);
log_debug("Rebuilding parent cache for " . count($artefacts) . " artefacts (users $i-$lastuser of $maxuser)");
rebuild_artefact_parent_cache_partial($artefacts);
}
$i += $userlimit;
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment