Commit 5c57b565 authored by Robert Lyon's avatar Robert Lyon

The archiving of submitted pages/collections from groups (Bug #1335670)

This patch contains:
- The export queue system where pages/collections on release from
submission are added to the export queue table ready to be archived.
- The export queue admin page showing what is in the queue to be
exported. The cron runs every 6 minutes. Queue items failed to export
are also shown here.
- The archive list admin page, where one can download the generated
leap2a files for the archived submissions.

In this patch you should be able to add things to the export queue by
either releasing a sumbission on a group that has 'archive
submissions' option ticked. This will add the archive to that archived
submission page, or you can also run a leap2a export from portfolio
export which will add the export queue and send you an email once the
export is done.

Things to note:
- The is a server busy function that stops the export queue from
running but I'm not too sure if the threshold is too low/high
- The export queue tries to export the first 100 items each run but if
resources are fine in handling that easily then the number could be
higher but I'm not sure of what will be a good number.
- Currently there is alsoe infrastructure like table columns for dealing
with releasing submissions from external systems (eg moodle) but that
functuionality is yet to be built.
- The checking of server busy in MS windows untested - may need to
just let MS ignore server busy check as there doesn't seem to be
standard way to check this.

Change-Id: If4c1d272e9c5d46fbf16b2ff73ceb2687c06ffd4
Signed-off-by: Robert Lyon's avatarRobert Lyon <robertl@catalyst.net.nz>
parent a91358cf
<?php
/**
*
* @package mahara
* @subpackage core
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
define('INTERNAL', 1);
define('JSON', 1);
define('INSTITUTIONALADMIN', 1);
require(dirname(dirname(dirname(__FILE__))) . '/init.php');
require_once('searchlib.php');
$params = new StdClass;
$params->query = trim(param_variable('query', ''));
$params->institution = param_alphanum('institution', null);
$params->sortby = param_alpha('sortby', 'firstname');
$params->sortdir = param_alpha('sortdir', 'asc');
$offset = param_integer('offset', 0);
$limit = param_integer('limit', 10);
list($html, $columns, $pagination, $search) = build_admin_archived_submissions_results($params, $offset, $limit);
json_reply(false, array(
'message' => null,
'data' => array(
'tablerows' => $html,
'pagination' => $pagination['html'],
'pagination_js' => $pagination['javascript']
)
));
<?php
/**
*
* @package mahara
* @subpackage admin
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
define('INTERNAL', 1);
define('INSTITUTIONALADMIN', 1);
define('MENUITEM', 'managegroups/archives');
require(dirname(dirname(dirname(__FILE__))) . '/init.php');
define('TITLE', get_string('archivedsubmissions', 'admin'));
define('SECTION_PLUGINTYPE', 'core');
define('SECTION_PLUGINNAME', 'admin');
define('SECTION_PAGE', 'archives');
require_once('searchlib.php');
$search = (object) array(
'query' => trim(param_variable('query', '')),
'sortby' => param_alpha('sortby', 'firstname'),
'sortdir' => param_alpha('sortdir', 'asc'),
);
$offset = param_integer('offset', 0);
$limit = param_integer('limit', 10);
if ($USER->get('admin')) {
$institutions = get_records_array('institution', '', '', 'displayname');
$search->institution = param_alphanum('institution', 'all');
}
else {
$institutionnames = array_keys($USER->get('admininstitutions'));
$institutions = get_records_select_array(
'institution',
'suspended = 0 AND name IN (' . join(',', array_fill(0, count($institutionnames), '?')) . ')',
$institutionnames,
'displayname'
);
}
list($html, $columns, $pagination, $search) = build_admin_archived_submissions_results($search, $offset, $limit);
$js = <<<EOF
addLoadEvent(function() {
var p = {$pagination['javascript']}
new UserSearch(p);
})
EOF;
$smarty = smarty(array('adminexportqueue','paginator'), array(), array('ascending' => 'mahara', 'descending' => 'mahara'));
$smarty->assign('search', $search);
$smarty->assign('limit', $limit);
$smarty->assign('institutions', $institutions);
$smarty->assign('results', $html);
$smarty->assign('pagination', $pagination['html']);
$smarty->assign('columns', $columns);
$smarty->assign('searchurl', $search['url']);
$smarty->assign('sortby', $search['sortby']);
$smarty->assign('sortdir', $search['sortdir']);
$smarty->assign('INLINEJAVASCRIPT', $js);
$smarty->assign('PAGEHEADING', TITLE);
$smarty->display('admin/groups/archives.tpl');
\ No newline at end of file
<?php
/**
*
* @package mahara
* @subpackage admin
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
define('INTERNAL', 1);
define('ADMIN', 1);
define('INSTITUTIONALADMIN', 1);
require(dirname(dirname(dirname(__FILE__))) . '/init.php');
require_once('searchlib.php');
$search = (object) array(
'query' => '',
'sortby' => 'firstname',
'sortdir' => 'asc',
'archivedsubmissions' => true,
);
$search->institution = param_alphanum('institution', null);
if (!empty($search->institution)) {
if (!$USER->get('admin') && !$USER->is_institutional_admin($search->institution)) {
throw new AccessDeniedException();
}
}
$results = get_admin_user_search_results($search, 0, false);
if (!empty($results['data'])) {
$csvfields = array('username', 'email', 'firstname', 'lastname', 'preferredname', 'submittedto', 'specialid', 'filetitle', 'filepath', 'filename', 'archivectime');
$USER->set_download_file(generate_csv($results['data'], $csvfields), 'archivedsubmissions.csv', 'text/csv');
redirect(get_config('wwwroot') . 'download.php');
}
$SESSION->add_error_msg(get_string('nocsvresults', 'admin'));
redirect(get_config('wwwroot') . 'admin/groups/archives.php?institution=' . $search->institution);
\ No newline at end of file
......@@ -211,6 +211,13 @@ $siteoptionform = array(
'defaultvalue' => get_config('showprogressbar'),
'disabled' => in_array('showprogressbar', $OVERRIDDEN),
),
'exporttoqueue' => array(
'type' => 'checkbox',
'title' => get_string('exporttoqueue', 'admin'),
'description' => get_string('exporttoqueuedescription', 'admin'),
'defaultvalue' => get_config('exporttoqueue'),
'disabled' => in_array('exporttoqueue', $OVERRIDDEN),
),
),
),
'searchsettings' => array(
......@@ -751,7 +758,7 @@ function siteoptions_submit(Pieform $form, $values) {
'registerterms', 'licensemetadata', 'licenseallowcustom', 'allowmobileuploads', 'creategroups', 'createpublicgroups', 'allowgroupcategories', 'wysiwyg',
'staffreports', 'staffstats', 'userscandisabledevicedetection', 'watchlistnotification_delay',
'masqueradingreasonrequired', 'masqueradingnotified', 'searchuserspublic',
'eventloglevel', 'eventlogexpiry', 'sitefilesaccess',
'eventloglevel', 'eventlogexpiry', 'sitefilesaccess', 'exporttoqueue',
);
$count = 0;
$where_sql = " WHERE admin = 0 AND id != 0";
......
......@@ -623,6 +623,12 @@ function edituser_delete_validate(Pieform $form, $values) {
$form->set_error('submit', get_string('deletefailed', 'admin'));
$SESSION->add_error_msg(get_string('deletefailed', 'admin'));
}
// Check to see if there are any pending archives in the export_queue for this user.
// We can't delete them if there are.
if ($results = count_records('export_queue', 'usr', $values['id'])) {
$form->set_error('submit', get_string('deletefailed', 'admin'));
$SESSION->add_error_msg(get_string('exportqueuenotempty', 'export'));
}
}
function edituser_delete_submit(Pieform $form, $values) {
......
<?php
/**
*
* @package mahara
* @subpackage core
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
define('INTERNAL', 1);
define('JSON', 1);
define('INSTITUTIONALADMIN', 1);
require(dirname(dirname(dirname(__FILE__))) . '/init.php');
require_once('searchlib.php');
$params = new StdClass;
$params->query = trim(param_variable('query', ''));
$params->institution = param_alphanum('institution', null);
$params->sortby = param_alpha('sortby', 'firstname');
$params->sortdir = param_alpha('sortdir', 'asc');
$offset = param_integer('offset', 0);
$limit = param_integer('limit', 10);
list($html, $columns, $pagination, $search) = build_admin_export_queue_results($params, $offset, $limit);
json_reply(false, array(
'message' => null,
'data' => array(
'tablerows' => $html,
'pagination' => $pagination['html'],
'pagination_js' => $pagination['javascript']
)
));
<?php
/**
*
* @package mahara
* @subpackage admin
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
define('INTERNAL', 1);
define('INSTITUTIONALADMIN', 1);
define('MENUITEM', 'configusers/exportqueue');
require(dirname(dirname(dirname(__FILE__))) . '/init.php');
define('TITLE', get_string('exportqueue', 'admin'));
define('SECTION_PLUGINTYPE', 'core');
define('SECTION_PLUGINNAME', 'admin');
define('SECTION_PAGE', 'exportqueue');
require_once('searchlib.php');
// If an action button has been submitted
if ($action = param_alphanum('action', null)) {
$rows = array();
if (param_variable('exportrows', null) && $action == 'export') {
$rows = param_variable('exportrows', null);
}
if (param_variable('deleterows', null) && $action == 'delete') {
$rows = param_variable('deleterows', null);
}
$rowids = array_map('intval', $rows);
// If only institutional admin make sure we can only action items that belong to users in our institution
if (!$USER->get('admin') && $USER->is_institutional_admin()) {
$institutions = $USER->get('admininstitutions');
foreach ($rowids as $key => $rowid) {
if (!get_field_sql('
SELECT COUNT(*) FROM {export_queue} e
JOIN {usr} u ON e.usr = u.id
JOIN {usr_institution} ui ON ui.usr = u.id
WHERE ui.institution IN (' . join(',', array_map('db_quote', $institutions)) . ')
AND e.id = ?',
array($rowid))) {
// not allowed to action this one.
unset($rowids[$key]);
}
}
}
// For deleting rows
if ($action == 'delete' && !empty($rowids)) {
foreach ($rowids as $rowid) {
db_begin();
// Need to relese any pending archiving
$items = get_records_select_array('export_queue_items', 'exportqueueid = ?', array($rowid), 'id');
$views = array();
// To make sure we process the item with this id only once we keep a track of the $lastid
// We don't know if the $item will be a collection or view (or artefact possibly in the future)
// In the case of a user exporting to leap2a there can be a number of collections/views to deal
// with so we want to deal with each collection or view only once.
$lastid = '';
$submitted = false;
$what = false;
foreach ($items as $key => $item) {
if (!empty($item->collection) && $lastid != 'collection_' . $item->collection) {
$what = 'collections';
$lastid = 'collection_' . $item->collection;
$views = array_merge($views, get_column('collection_view', 'view', 'collection', $item->collection));
$submitted = get_record('collection', 'id', $item->collection);
}
else if (empty($item->collection) && !empty($item->view) && $lastid != 'view_' . $item->view) {
$what = 'views';
$lastid = 'view_' . $item->view;
$views = array_merge($views, array($item->view));
$submitted = get_record('view', 'id', $item->view);
}
}
require_once(get_config('docroot') . 'lib/view.php');
if ($submitted->submittedstatus == View::PENDING_RELEASE) {
// we need to release the submission
if ($what == 'collections') {
require_once(get_config('docroot') . 'lib/collection.php');
$id = substr($lastid, strlen('collection_'));
$collection = new Collection($id);
try {
$collection->release($USER->get('id'));
}
catch (SystemException $e) {
$errors[] = get_string('submissionreleasefailed', 'export');
log_warn($e->getMessage());
}
}
else if ($what == 'views') {
$id = substr($lastid, strlen('view_'));
$view = new View($id);
try {
$view->release($USER->get('id'));
}
catch (SystemException $e) {
$errors[] = get_string('submissionreleasefailed', 'export');
log_warn($e->getMessage());
}
}
else {
$errors[] = get_string('submissionreleasefailed', 'export');
}
}
if (!delete_records('export_queue_items', 'exportqueueid', $rowid)) {
log_warn('Unable to delete export queue items for ID: ' . $rowid);
db_rollback();
}
if (!delete_records('export_queue', 'id', $rowid)) {
log_warn('Unable to delete export queue row ID: ' . $rowid);
db_rollback();
}
db_commit();
}
$SESSION->add_ok_msg(get_string('exportqueuedeleted', 'admin', count($rowids)));
}
else if ($action == 'export' && !empty($rowids)) {
// Make failed rows change to pending rows so they can be picked up next cron run
foreach ($rowids as $rowid) {
execute_sql('UPDATE {export_queue} SET starttime = NULL, ctime = NOW() WHERE id = ?', array($rowid));
}
$SESSION->add_ok_msg(get_string('exportqueuearchived', 'admin', count($rowids)));
}
}
$search = (object) array(
'query' => trim(param_variable('query', '')),
'sortby' => param_alpha('sortby', 'firstname'),
'sortdir' => param_alpha('sortdir', 'asc'),
);
$offset = param_integer('offset', 0);
$limit = param_integer('limit', 10);
if ($USER->get('admin')) {
$institutions = get_records_array('institution', '', '', 'displayname');
$search->institution = param_alphanum('institution', 'all');
}
else {
$institutionnames = array_keys($USER->get('admininstitutions'));
$institutions = get_records_select_array(
'institution',
'suspended = 0 AND name IN (' . join(',', array_fill(0, count($institutionnames), '?')) . ')',
$institutionnames,
'displayname'
);
}
list($html, $columns, $pagination, $search) = build_admin_export_queue_results($search, $offset, $limit);
$js = <<<EOF
addLoadEvent(function() {
var p = {$pagination['javascript']}
new UserSearch(p);
})
EOF;
$smarty = smarty(array('adminexportqueue', 'paginator'), array(), array('ascending' => 'mahara', 'descending' => 'mahara'));
$smarty->assign('search', $search);
$smarty->assign('limit', $limit);
$smarty->assign('institutions', $institutions);
$smarty->assign('results', $html);
$smarty->assign('pagination', $pagination['html']);
$smarty->assign('columns', $columns);
$smarty->assign('searchurl', $search['url']);
$smarty->assign('sortby', $search['sortby']);
$smarty->assign('sortdir', $search['sortdir']);
$smarty->assign('INLINEJAVASCRIPT', $js);
$smarty->assign('PAGEHEADING', TITLE);
$smarty->display('admin/users/exportqueue.tpl');
......@@ -1004,6 +1004,69 @@ class User {
return false;
}
/**
* Check if user can download/view an export archive. Will return true:
* if the user is the owner of the archive, or
* if the user is a site admin, or
* if the user is a group admin of the group the collection/view was submitted to, or
* if the user is an admin of the institution that the group belongs to, or
* if the user is an institutional admin of any institutions that the submitter belongs to
*
* @param $data Record containing information from the export_archive and archived_submission tables
*
* @return bool
*/
function can_view_archive($data) {
global $USER;
require_once(get_config('docroot') . 'auth/lib.php');
$user = new User;
$user->find_by_id($data->usr);
// User is the owner of the archive so is allowed to see it
if ($USER->get('id') == $user->get('id')) {
return true;
}
// User is a site admin so is allowed to access everything
if ($USER->get('admin')) {
return true;
}
if (!empty($data->group)) {
// User is a group admin of the group the collection/view was submitted to
$grouproles = $USER->get('grouproles');
if (!empty($grouproles[$data->group]) && $grouproles[$data->group] == 'admin') {
return true;
}
// User is an institutional admin for the institution that the group belongs to
// Currently only groups uploaded via csv can get the institution field set.
$currentuserinstitutions = $USER->get('institutions');
$groupinstitution = get_field('group','institution', 'id', $data->group);
if (!empty($groupinstitution)) {
foreach ($currentuserinstitutions as $key => $institution) {
if ($USER->is_institutional_admin($key) && $key == $groupinstitution) {
return true;
}
}
}
}
// User is an institutional admin in an institution that the data->usr belongs to
// This is a loose connection check for groups without the institution field set.
// But seen as the User has power over the data->usr we will allow it
$ownerinstitutions = $user->get('institutions');
$currentuserinstitutions = $USER->get('institutions');
foreach ($currentuserinstitutions as $key => $institution) {
if ($USER->is_institutional_admin($key) && !empty($ownerinstitutions[$key])) {
return true;
}
}
return false;
}
/**
* Indicates whether the user has permission to edit an artefact's contents. The name refers
* to the "edit" permission for group files.
......
......@@ -2,6 +2,7 @@
<div class="{cycle values='r0,r1'} listrow">
<h4 class="title"><a href="{$item.url}">{$item.name|str_shorten_text:60:true}</a>
<span class="owner">{str tag=by section=view} <a href="{$item.ownerurl}">{$item.ownername}</a></span></h4>
<div class="detail">{str tag=timeofsubmission section=view}: {$item.submittedtime|format_date}</div>
{* submittedstatus == '2' is equivalent to PENDING_RELEASE *}
<div class="detail">{str tag=timeofsubmission section=view}: {$item.submittedtime|format_date} {if $item.submittedstatus == '2'}- {str tag=submittedpendingrelease section=view}{/if}</div>
</div>
{/foreach}
......@@ -74,6 +74,8 @@
{else}
{str tag=youhavesubmitted section=view arg1=$item.url arg2=$item.name}
{/if}
{* submittedstatus == '2' is equivalent to PENDING_RELEASE *}
{if $item.submittedstatus == '2'}- {str tag=submittedpendingrelease section=view}{/if}
</div>
{/foreach}
{/if}
......
<?php
/**
*
* @package mahara
* @subpackage core
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
define('INTERNAL', 1);
define('PUBLIC', 1);
require('init.php');
require_once('file.php');
$id = param_integer('id');
$data = get_record_sql('SELECT e.*, a.* FROM {export_archive} e
LEFT JOIN {archived_submissions} a ON a.archiveid = e.id
WHERE e.id = ?', array($id));
if (!$USER->is_logged_in()) {
throw new AccessDeniedException();
}
if (empty($data) || empty($data->filename) || !$USER->can_view_archive($data)) {
throw new AccessDeniedException();
}
$path = $data->filepath . $data->filename;
$name = $data->filename;
$mimetype = 'application/zip';
if (!file_exists($path)) {
throw new NotFoundException(get_string('filenotfoundmaybeexpired'));
}
serve_file($path, $name, $mimetype);
......@@ -87,6 +87,12 @@ default:
}
$exporter->includefeedback = $exportdata['includefeedback'];
// Get an estimate of how big the unzipped export file would be
// so we can check that we have enough disk space for it
$space = $exporter->is_diskspace_available();
if (!$space) {
export_iframe_die(get_string('exportfiletoobig', 'mahara'), get_config('wwwroot') . 'view/index.php');
}
try {
$zipfile = $exporter->export();
......
......@@ -101,6 +101,10 @@ class PluginExportHtml extends PluginExport {
return get_string('description', 'export.html');
}
public function is_diskspace_available() {
return true; // need to create a check here
}
/**
* Main export routine
*/
......
......@@ -144,8 +144,9 @@ function export_validate(Pieform $form, $values) {
}
function export_submit(Pieform $form, $values) {
global $SESSION;
global $SESSION, $USER;
$views = array();
$collections = array();
if ($values['what'] == 'views') {
foreach ($values as $key => $value) {
if (substr($key, 0, 5) == 'view_' && $value) {
......@@ -157,23 +158,46 @@ function export_submit(Pieform $form, $values) {
foreach ($values as $key => $value) {
if (substr($key, 0, 11) == 'collection_' && $value) {
$collection = intval(substr($key, 11));
$collections[] = $collection;
$views = array_merge($views, get_column('collection_view', 'view', 'collection', $collection));
}
}
}
$exportdata = array(
'format' => $values['format'],
'what' => $values['what'],
'views' => $views,
'includefeedback' => $values['includefeedback'],
);
$SESSION->set('exportdata', $exportdata);
if ($values['format'] == 'leap' && get_config('exporttoqueue') == 1) {
// insert into the export_queue;
require_once(get_config('docroot') . 'export/lib.php');
$objectarray = array();
if ($values['what'] == 'collections') {
foreach ($collections as $collectionid) {
$collection = new Collection($collectionid);
$objectarray[] = $collection;
}
}
else if ($values['what'] == 'views') {
foreach ($views as $viewid) {
$view = new View($viewid);
$objectarray[] = $view;
}
}
export_add_to_queue($objectarray, null, $USER, $values['what']);
$SESSION->add_ok_msg(get_string('addedleap2atoexportqueue' . $values['what'], 'export'));
redirect('/export/index.php');
}
else {
$exportdata = array(
'format' => $values['format'],
'what' => $values['what'],
'views' => $views,
'includefeedback' => $values['includefeedback'],