Commit 55a8deb8 authored by Nigel Cunningham's avatar Nigel Cunningham Committed by Gerrit Code Review
Browse files

(Bug1352028) Add a JSON progress bar for bulk operations.



This patch adds a JSON progress meter (I'll call it that to avoid confusion
with progress bars) to the bulk uploading of users, groups and group
memberships and the bulk export and import of users (LEAP), so the user can see
the progress of the operation and not just the submit button changed to
'Processing..' and whatever indication their browser gives while waiting for
content.

The bulk export and import are minor rewrites, replacing the old iframe based
progress bar and the associated multiple pages and additional template file in
the case of the bulk export, and the recursive redirect-to-self of the bulk
import.

To accomplish the display of the progress bar during the operation, we make the
PHP session be closed (read only) except when changes need to be made. This is
for the most part a straightforward change in session.php as it's the only
direct accessor. In other places, we replace direct accessing of the session
variable ($_SESSION) with use of the session class ($SESSION) so that it can
reopen the session, make the change and close the session again.

There is one more aspect to all of this: with previous behaviour, multiple
requests for the same session would queue, taking the session lock in turn.
After this patch is applied, they can proceed in parallel, allowing greater
throughput. There is no additional locking requirement because the issues are
the same as those already dealt with in allowing multiple PHP threads to
process requests from different sessions at the same time.

I have sought to make the progress meter nice and generic, so it can be used in
the other bulk imports and exports too.

Paradoxically, these changes don't just make the import seem to be faster, it
actually is.. at least in the case of users and groups.

Times for importing 1000 users, groups and memberships, averaged over 3 runs
each (Wall time, not CPU time - but the relationship is the same).

                Without Progress     With Progress
Users                166s               155s
Groups                85s                78s
Memberships           20s                19s

Change-Id: Iec15c57db32c77994edb80c71d65591de51a95e4
Signed-off-by: default avatarNigel Cunningham <nigelc@catalyst-au.net>
parent a596b554
......@@ -74,6 +74,11 @@ $form = array(
'description' => get_string('updategroupsdescription', 'admin'),
'defaultvalue' => false,
),
'progress_meter_token' => array(
'type' => 'hidden',
'value' => 'uploadgroupscsv',
'readonly' => TRUE,
),
'submit' => array(
'type' => 'submit',
'value' => get_string('uploadgroupcsv', 'admin')
......@@ -116,6 +121,7 @@ function uploadcsv_validate(Pieform $form, $values) {
$csvgroups->set('mandatoryfields', $MANDATORYFIELDS);
$csvdata = $csvgroups->get_data();
$num_lines = count($csvdata->data);
if (!empty($csvdata->errors['file'])) {
$form->set_error('file', $csvdata->errors['file']);
......@@ -133,6 +139,11 @@ function uploadcsv_validate(Pieform $form, $values) {
// If headers exists, increment i = key + 2 for actual line number
$i = ($csvgroups->get('headerExists')) ? ($key + 2) : ($key + 1);
// In adding 5000 groups, this part was approx 10% of the wall time.
if (!($key % 25)) {
set_progress_info('uploadgroupscsv', $key, $num_lines * 10, get_string('validating', 'admin'));
}
// Trim non-breaking spaces -- they get left in place by File_CSV
foreach ($line as &$field) {
$field = preg_replace('/^(\s|\xc2\xa0)*(.*?)(\s|\xc2\xa0)*$/', '$2', $field);
......@@ -266,8 +277,16 @@ function uploadcsv_submit(Pieform $form, $values) {
$addedgroups = array();
$key = 0;
$num_lines = count($CSVDATA);
foreach ($CSVDATA as $record) {
if (!($key % 25)) {
set_progress_info('uploadgroupscsv', $num_lines + $key * 9, $num_lines * 10, get_string('committingchanges', 'admin'));
}
$key++;
$group = new StdClass;
$group->name = $record[$formatkeylookup['displayname']];
$group->shortname = $record[$formatkeylookup['shortname']];
......@@ -323,6 +342,9 @@ function uploadcsv_submit(Pieform $form, $values) {
else {
$SESSION->add_ok_msg(get_string('numbernewgroupsadded', 'admin', count($addedgroups)));
}
set_progress_done('uploadgroupscsv');
redirect('/admin/groups/uploadcsv.php');
}
......@@ -353,6 +375,8 @@ $uploadcsvpagedescription = get_string('uploadgroupcsvpagedescription2', 'admin'
$form = pieform($form);
set_progress_done('uploadgroupscsv');
$smarty = smarty(array('adminuploadcsv'));
$smarty->assign('uploadcsvpagedescription', $uploadcsvpagedescription);
$smarty->assign('uploadcsvform', $form);
......
......@@ -50,6 +50,11 @@ $form = array(
'required' => true
)
),
'progress_meter_token' => array(
'type' => 'hidden',
'value' => 'uploadgroupmemberscsv',
'readonly' => TRUE,
),
'submit' => array(
'type' => 'submit',
'value' => get_string('uploadgroupmemberscsv', 'admin')
......@@ -105,10 +110,17 @@ function uploadcsv_validate(Pieform $form, $values) {
$shortnames = array();
$hadadmin = array();
$num_lines = count($csvdata->data);
foreach ($csvdata->data as $key => $line) {
// If headers exists, increment i = key + 2 for actual line number
$i = ($csvgroups->get('headerExists')) ? ($key + 2) : ($key + 1);
// In adding 5000 groups, this part was approx 8% of the wall time.
if (!($key % 25)) {
set_progress_info('uploadgroupmemberscsv', $key, $num_lines * 10, get_string('validating', 'admin'));
}
// Trim non-breaking spaces -- they get left in place by File_CSV
foreach ($line as &$field) {
$field = preg_replace('/^(\s|\xc2\xa0)*(.*?)(\s|\xc2\xa0)*$/', '$2', $field);
......@@ -186,9 +198,12 @@ function uploadcsv_submit(Pieform $form, $values) {
db_begin();
$lines_done = 0;
$num_lines = count($CSVDATA);
foreach ($MEMBERS as $gid => $members) {
$updates = group_update_members($gid, $members);
$updates = group_update_members($gid, $members, $lines_done, $num_lines);
$lines_done += sizeof($members);
if (empty($updates)) {
unset($UPDATES[$GROUPS[$gid]]);
......@@ -211,6 +226,7 @@ function uploadcsv_submit(Pieform $form, $values) {
else {
$SESSION->add_ok_msg(get_string('numbergroupsupdated', 'admin', 0));
}
set_progress_done('uploadgroupmemberscsv');
redirect('/admin/groups/uploadmemberscsv.php');
}
......@@ -220,6 +236,8 @@ $uploadcsvpagedescription = get_string('uploadgroupmemberscsvpagedescription3',
$form = pieform($form);
set_progress_done('uploadgroupmemberscsv');
$smarty = smarty(array('adminuploadcsv'));
$smarty->assign('uploadcsvpagedescription', $uploadcsvpagedescription);
$smarty->assign('uploadcsvform', $form);
......
<?php
/**
*
* @package mahara
* @subpackage export
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
define('INTERNAL', 1);
define('ADMIN', 1);
define('BULKEXPORT', 1);
require(dirname(dirname(dirname(__FILE__))) . '/init.php');
require_once(get_config('docroot') . '/lib/htmloutput.php');
raise_memory_limit("1024M");
raise_time_limit(300);
// Download the export file if it's been generated
if ($exportfile = $SESSION->get('exportfile')) {
$SESSION->set('exportdata', '');
$SESSION->set('exportfile', '');
require_once('file.php');
serve_file($exportfile, basename($exportfile), 'application/x-zip', array('lifetime' => 0, 'forcedownload' => true));
exit;
// TODO: delete the zipfile (and temporary files) once it's been downloaded
}
// Turn off all compression because it prevents output from being flushed
if (function_exists('apache_setenv')) {
apache_setenv('no-gzip', 1);
}
@ini_set('zlib.output_compression', 0);
if (!$exportdata = $SESSION->get('exportdata')) {
redirect(get_config('wwwroot').'admin/users/bulkexport.php');
}
$SESSION->set('exportdata', '');
$stylesheets = array_reverse($THEME->get_url('style/style.css', true));
print_export_head($stylesheets);
flush();
/**
* Outputs enough HTML to make a pretty error message in the iframe
*
* @param string $message The message to display to the user
*/
function export_iframe_die($message) {
print_export_iframe_die($message);
exit;
}
/**
* Registered as the progress report handler for the export. Streams updates
* back to the browser
*
* @param int $percent How far complete the export is
* @param string $status A human-readable string describing the current step
*/
function export_iframe_progress_handler($percent, $status) {
print_iframe_progress_handler($percent, $status);
ob_flush();
}
/**
* Convert a 2D array to a CSV file. This follows the basic rules from http://en.wikipedia.org/wiki/Comma-separated_values
*
* @param array $input 2D array of values: each line is an array of values
*/
function data_to_csv($input) {
if (empty($input) or !is_array($input)) {
return '';
}
$output = '';
foreach ($input as $line) {
$lineoutput = '';
foreach ($line as $element) {
$element = str_replace('"', '""', $element);
if (!empty($lineoutput)) {
$lineoutput .= ',';
}
$lineoutput .= "\"$element\"";
}
$output .= $lineoutput . "\r\n";
}
return $output;
}
function create_zipfile($listing, $files) {
global $USER;
if (empty($listing) or empty($files)) {
return false;
}
if (count($listing) != count($files)) {
throw new MaharaException("Files and listing don't match.");
}
// create temporary directories for the export
$exportdir = get_config('dataroot') . 'export/'
. $USER->get('id') . '/' . time() . '/';
if (!check_dir_exists($exportdir)) {
throw new SystemException("Couldn't create the temporary export directory $exportdir");
}
$usersdir = 'users/';
if (!check_dir_exists($exportdir . $usersdir)) {
throw new SystemException("Couldn't create the temporary export directory $usersdir");
}
// move user zipfiles into the export directory
foreach ($files as $filename) {
if (copy($filename, $exportdir . $usersdir . basename($filename))) {
unlink($filename);
}
else {
throw new SystemException("Couldn't move $filename to $usersdir");
}
}
// write username listing to a file
$listingfile = 'usernames.csv';
if (!file_put_contents($exportdir . $listingfile, data_to_csv($listing))) {
throw new SystemException("Couldn't write usernames to a file");
}
// zip everything up
$zipfile = $exportdir . 'mahara-bulk-export-' . time() . '.zip';
$cwd = getcwd();
$command = sprintf('%s %s %s %s %s',
get_config('pathtozip'),
get_config('ziprecursearg'),
escapeshellarg($zipfile),
escapeshellarg($listingfile),
escapeshellarg($usersdir)
);
$output = array();
chdir($exportdir);
exec($command, $output, $returnvar);
chdir($cwd);
if ($returnvar != 0) {
throw new SystemException('Failed to zip the export file: return code ' . $returnvar);
}
return $zipfile;
}
// Bail if we don't have enough data to do an export
if (empty($exportdata)) {
export_iframe_die(get_string('unabletogenerateexport', 'export'));
}
ob_start();
export_iframe_progress_handler(0, get_string('Setup', 'export'));
safe_require('export', 'leap');
$listing = array();
$files = array();
$exportcount = 0;
$exporterrors = array();
foreach ($exportdata as $username) {
$user = new User();
try {
$user->find_by_username($username);
} catch (AuthUnknownUserException $e) {
continue; // Skip non-existent users
}
$percentage = (double)$exportcount / count($exportdata) * 100;
$percentage = min($percentage, 98);
export_iframe_progress_handler($percentage, get_string('exportingusername', 'admin', $username));
$exporter = new PluginExportLeap($user, PluginExport::EXPORT_ALL_VIEWS, PluginExport::EXPORT_ALL_ARTEFACTS);
try {
$zipfile = $exporter->export();
} catch (Exception $e) {
$exporterrors[] = $username;
continue;
}
$listing[] = array($username, $zipfile);
$files[] = $exporter->get('exportdir') . $zipfile;
$exportcount++;
}
export_iframe_progress_handler(99, get_string('creatingzipfile', 'export'));
if (!$zipfile = create_zipfile($listing, $files)) {
export_iframe_die(get_string('bulkexportempty', 'admin'));
}
export_iframe_progress_handler(100, get_string('Done', 'export'));
ob_end_flush();
log_info("Exported $exportcount users to $zipfile");
if (!empty($exporterrors)) {
$SESSION->add_error_msg(get_string('couldnotexportusers', 'admin', implode(', ', $exporterrors)));
}
// Store the filename in the session, and redirect the iframe to it to trigger
// the download. Here it would be nice to trigger the download for everyone,
// but alas this is not possible for people without javascript.
$SESSION->set('exportfile', $zipfile);
$continueurljs = get_config('wwwroot');
$continueurl = 'bulkdownload.php';
$result = $SESSION->get('messages');
$SESSION->clear('messages');
print_export_footer(get_string('exportgeneratedsuccessfully1', 'export'), $continueurl, $continueurljs, $result, 'bulkdownload.php');
......@@ -16,6 +16,98 @@ require_once('pieforms/pieform.php');
define('TITLE', get_string('bulkexporttitle', 'admin'));
/**
* Convert a 2D array to a CSV file. This follows the basic rules from http://en.wikipedia.org/wiki/Comma-separated_values
*
* @param array $input 2D array of values: each line is an array of values
*/
function data_to_csv($input) {
if (empty($input) or !is_array($input)) {
return '';
}
$output = '';
foreach ($input as $line) {
$lineoutput = '';
foreach ($line as $element) {
$element = str_replace('"', '""', $element);
if (!empty($lineoutput)) {
$lineoutput .= ',';
}
$lineoutput .= "\"$element\"";
}
$output .= $lineoutput . "\r\n";
}
return $output;
}
/**
* Create a zip archive containing the exported data.
*
* @param array $listing The list of usernames that were exported
* @param array $files A list of archive files for each user
*/
function create_zipfile($listing, $files) {
global $USER;
if (empty($listing) or empty($files)) {
return false;
}
if (count($listing) != count($files)) {
throw new MaharaException("Files and listing don't match.");
}
// create temporary directories for the export
$exportdir = get_config('dataroot') . 'export/'
. $USER->get('id') . '/' . time() . '/';
if (!check_dir_exists($exportdir)) {
throw new SystemException("Couldn't create the temporary export directory $exportdir");
}
$usersdir = 'users/';
if (!check_dir_exists($exportdir . $usersdir)) {
throw new SystemException("Couldn't create the temporary export directory $usersdir");
}
// move user zipfiles into the export directory
foreach ($files as $filename) {
if (copy($filename, $exportdir . $usersdir . basename($filename))) {
unlink($filename);
}
else {
throw new SystemException("Couldn't move $filename to $usersdir");
}
}
// write username listing to a file
$listingfile = 'usernames.csv';
if (!file_put_contents($exportdir . $listingfile, data_to_csv($listing))) {
throw new SystemException("Couldn't write usernames to a file");
}
// zip everything up
$zipfile = $exportdir . 'mahara-bulk-export-' . time() . '.zip';
$cwd = getcwd();
$command = sprintf('%s %s %s %s %s',
get_config('pathtozip'),
get_config('ziprecursearg'),
escapeshellarg($zipfile),
escapeshellarg($listingfile),
escapeshellarg($usersdir)
);
$output = array();
chdir($exportdir);
exec($command, $output, $returnvar);
chdir($cwd);
if ($returnvar != 0) {
throw new SystemException('Failed to zip the export file: return code ' . $returnvar);
}
return $zipfile;
}
function bulkexport_submit(Pieform $form, $values) {
global $SESSION;
......@@ -37,12 +129,62 @@ function bulkexport_submit(Pieform $form, $values) {
}
}
$SESSION->set('exportdata', $usernames);
safe_require('export', 'leap');
$listing = array();
$files = array();
$exportcount = 0;
$exporterrors = array();
$num_users = count($usernames);
foreach ($usernames as $username) {
if (!($exportcount % 25)) {
set_progress_info('bulkexport', $exportcount, $num_users, get_string('validating', 'admin'));
}
$user = new User();
try {
$user->find_by_username($username);
}
catch (AuthUnknownUserException $e) {
continue; // Skip non-existent users
}
$exporter = new PluginExportLeap($user, PluginExport::EXPORT_ALL_VIEWS, PluginExport::EXPORT_ALL_ARTEFACTS);
try {
$zipfile = $exporter->export();
}
catch (Exception $e) {
$exporterrors[] = $username;
continue;
}
$listing[] = array($username, $zipfile);
$files[] = $exporter->get('exportdir') . $zipfile;
$exportcount++;
}
if (!$zipfile = create_zipfile($listing, $files)) {
export_iframe_die(get_string('bulkexportempty', 'admin'));
}
$smarty = smarty();
$smarty->assign('heading', '');
$smarty->display('admin/users/bulkdownload.tpl');
exit;
log_info("Exported $exportcount users to $zipfile");
if (!empty($exporterrors)) {
$SESSION->add_error_msg(get_string('couldnotexportusers', 'admin', implode(', ', $exporterrors)));
}
// Store the filename in the session, and redirect the iframe to it to trigger
// the download. Here it would be nice to trigger the download for everyone,
// but alas this is not possible for people without javascript.
$SESSION->set('exportfile', $zipfile);
set_progress_done('bulkexport', array('redirect' => '/admin/users/bulkexport.php'));
// Download the export file once it has been generated
require_once('file.php');
serve_file($zipfile, basename($zipfile), 'application/x-zip', array('lifetime' => 0, 'forcedownload' => true));
// TODO: delete the zipfile (and temporary files) once it's been downloaded
}
$authinstanceelement = array('type' => 'hidden', 'value' => '');
......@@ -76,6 +218,11 @@ $form = array(
'title' => get_string('bulkexportusernames', 'admin'),
'description' => get_string('bulkexportusernamesdescription', 'admin'),
),
'progress_meter_token' => array(
'type' => 'hidden',
'value' => 'bulkexport',
'readonly' => TRUE,
),
'submit' => array(
'type' => 'submit',
'value' => get_string('bulkexport', 'admin')
......@@ -83,6 +230,8 @@ $form = array(
)
);
set_progress_done('bulkexport');
$form = pieform($form);
$smarty = smarty();
......
......@@ -25,29 +25,6 @@ define('TITLE', get_string('bulkleap2aimport', 'admin'));
// Turn on autodetecting of line endings, so mac newlines (\r) will work
ini_set('auto_detect_line_endings', 1);
$ADDEDUSERS = $SESSION->get('bulkimport_addedusers');
if (empty($ADDEDUSERS)) {
$ADDEDUSERS = array();
}
$FAILEDUSERS = $SESSION->get('bulkimport_failedusers');
if (empty($FAILEDUSERS)) {
$FAILEDUSERS = array();
}
$LEAP2AFILES = $SESSION->get('bulkimport_leap2afiles');
if (empty($LEAP2AFILES)) {
$LEAP2AFILES = array();
}
$AUTHINSTANCE = $SESSION->get('bulkimport_authinstance');
$EMAILUSERS = $SESSION->get('bulkimport_emailusers');
// Import in progress
if (!empty($LEAP2AFILES)) {
import_next_user();
}
elseif (!empty($ADDEDUSERS) or !empty($FAILEDUSERS)) {
finish_import();
}
$authinstances = auth_get_auth_instances();
if (count($authinstances) > 0) {
......@@ -86,6 +63,11 @@ $form = array(
'description' => get_string('emailusersaboutnewaccountdescription', 'admin'),
'defaultvalue' => true,
),
'progress_meter_token' => array(
'type' => 'hidden',
'value' => 'bulkimport',
'readonly' => TRUE,
),
'submit' => array(
'type' => 'submit',
'value' => get_string('Import', 'admin')
......@@ -93,25 +75,6 @@ $form = array(
)
);
/**
* Work-around the redirection limit of Firefox (http://kb.mozillazine.org/Network.http.redirection-limit)
*/
function meta_redirect() {
global $SESSION, $LEAP2AFILES, $ADDEDUSERS, $FAILEDUSERS;
$SESSION->set('bulkimport_leap2afiles', $LEAP2AFILES);
$SESSION->set('bulkimport_addedusers', $ADDEDUSERS);
$SESSION->set('bulkimport_failedusers', $FAILEDUSERS);
$url = get_config('wwwroot') . '/admin/users/bulkimport.php';
$failed = sizeof($FAILEDUSERS) ? ' (' . sizeof($FAILEDUSERS) . ' failed)' : '';
$done = sizeof($FAILEDUSERS) + sizeof($ADDEDUSERS);
$total = $done + sizeof($LEAP2AFILES);
$title = "Completed {$done}/{$total}{$failed}";
print_meta_redirect($url, $title);
exit;
}
/**
* The CSV file is parsed here so validation errors can be returned to the
* user. The data from a successful parsing is stored in the <var>$LEAP2AFILES</var>
......@@ -196,32 +159,34 @@ function bulkimport_validate(Pieform $form, $values) {
function bulkimport_submit(Pieform $form, $values) {
global $SESSION, $LEAP2AFILES;
log_info('Attempting to import ' . count($LEAP2AFILES) . ' users from Leap2A files');
require_once('file.php');
require_once(get_config('docroot') . 'import/lib.php');
safe_require('import', 'leap');
$key = 0;
$total = count($LEAP2AFILES);
$SESSION->set('bulkimport_leap2afiles', $LEAP2AFILES);