Commit fa461a5e authored by Robert Lyon's avatar Robert Lyon Committed by Gerrit Code Review
Browse files

Merge "(Bug1352028) Add a JSON progress bar for bulk operations."

parents 90352d45 55a8deb8
......@@ -74,6 +74,11 @@ $form = array(
'description' => get_string('updategroupsdescription', 'admin'),
'defaultvalue' => false,
),
'progress_meter_token' => array(
'type' => 'hidden',
'value' => 'uploadgroupscsv',
'readonly' => TRUE,
),
'submit' => array(
'type' => 'submit',
'value' => get_string('uploadgroupcsv', 'admin')
......@@ -116,6 +121,7 @@ function uploadcsv_validate(Pieform $form, $values) {
$csvgroups->set('mandatoryfields', $MANDATORYFIELDS);
$csvdata = $csvgroups->get_data();
$num_lines = count($csvdata->data);
if (!empty($csvdata->errors['file'])) {
$form->set_error('file', $csvdata->errors['file']);
......@@ -133,6 +139,11 @@ function uploadcsv_validate(Pieform $form, $values) {
// If headers exists, increment i = key + 2 for actual line number
$i = ($csvgroups->get('headerExists')) ? ($key + 2) : ($key + 1);
// In adding 5000 groups, this part was approx 10% of the wall time.
if (!($key % 25)) {
set_progress_info('uploadgroupscsv', $key, $num_lines * 10, get_string('validating', 'admin'));
}
// Trim non-breaking spaces -- they get left in place by File_CSV
foreach ($line as &$field) {
$field = preg_replace('/^(\s|\xc2\xa0)*(.*?)(\s|\xc2\xa0)*$/', '$2', $field);
......@@ -266,8 +277,16 @@ function uploadcsv_submit(Pieform $form, $values) {
$addedgroups = array();
$key = 0;
$num_lines = count($CSVDATA);
foreach ($CSVDATA as $record) {
if (!($key % 25)) {
set_progress_info('uploadgroupscsv', $num_lines + $key * 9, $num_lines * 10, get_string('committingchanges', 'admin'));
}
$key++;
$group = new StdClass;
$group->name = $record[$formatkeylookup['displayname']];
$group->shortname = $record[$formatkeylookup['shortname']];
......@@ -323,6 +342,9 @@ function uploadcsv_submit(Pieform $form, $values) {
else {
$SESSION->add_ok_msg(get_string('numbernewgroupsadded', 'admin', count($addedgroups)));
}
set_progress_done('uploadgroupscsv');
redirect('/admin/groups/uploadcsv.php');
}
......@@ -353,6 +375,8 @@ $uploadcsvpagedescription = get_string('uploadgroupcsvpagedescription2', 'admin'
$form = pieform($form);
set_progress_done('uploadgroupscsv');
$smarty = smarty(array('adminuploadcsv'));
$smarty->assign('uploadcsvpagedescription', $uploadcsvpagedescription);
$smarty->assign('uploadcsvform', $form);
......
......@@ -50,6 +50,11 @@ $form = array(
'required' => true
)
),
'progress_meter_token' => array(
'type' => 'hidden',
'value' => 'uploadgroupmemberscsv',
'readonly' => TRUE,
),
'submit' => array(
'type' => 'submit',
'value' => get_string('uploadgroupmemberscsv', 'admin')
......@@ -105,10 +110,17 @@ function uploadcsv_validate(Pieform $form, $values) {
$shortnames = array();
$hadadmin = array();
$num_lines = count($csvdata->data);
foreach ($csvdata->data as $key => $line) {
// If headers exists, increment i = key + 2 for actual line number
$i = ($csvgroups->get('headerExists')) ? ($key + 2) : ($key + 1);
// In adding 5000 groups, this part was approx 8% of the wall time.
if (!($key % 25)) {
set_progress_info('uploadgroupmemberscsv', $key, $num_lines * 10, get_string('validating', 'admin'));
}
// Trim non-breaking spaces -- they get left in place by File_CSV
foreach ($line as &$field) {
$field = preg_replace('/^(\s|\xc2\xa0)*(.*?)(\s|\xc2\xa0)*$/', '$2', $field);
......@@ -186,9 +198,12 @@ function uploadcsv_submit(Pieform $form, $values) {
db_begin();
$lines_done = 0;
$num_lines = count($CSVDATA);
foreach ($MEMBERS as $gid => $members) {
$updates = group_update_members($gid, $members);
$updates = group_update_members($gid, $members, $lines_done, $num_lines);
$lines_done += sizeof($members);
if (empty($updates)) {
unset($UPDATES[$GROUPS[$gid]]);
......@@ -211,6 +226,7 @@ function uploadcsv_submit(Pieform $form, $values) {
else {
$SESSION->add_ok_msg(get_string('numbergroupsupdated', 'admin', 0));
}
set_progress_done('uploadgroupmemberscsv');
redirect('/admin/groups/uploadmemberscsv.php');
}
......@@ -220,6 +236,8 @@ $uploadcsvpagedescription = get_string('uploadgroupmemberscsvpagedescription3',
$form = pieform($form);
set_progress_done('uploadgroupmemberscsv');
$smarty = smarty(array('adminuploadcsv'));
$smarty->assign('uploadcsvpagedescription', $uploadcsvpagedescription);
$smarty->assign('uploadcsvform', $form);
......
<?php
/**
*
* @package mahara
* @subpackage export
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
define('INTERNAL', 1);
define('ADMIN', 1);
define('BULKEXPORT', 1);
require(dirname(dirname(dirname(__FILE__))) . '/init.php');
require_once(get_config('docroot') . '/lib/htmloutput.php');
raise_memory_limit("1024M");
raise_time_limit(300);
// Download the export file if it's been generated
if ($exportfile = $SESSION->get('exportfile')) {
$SESSION->set('exportdata', '');
$SESSION->set('exportfile', '');
require_once('file.php');
serve_file($exportfile, basename($exportfile), 'application/x-zip', array('lifetime' => 0, 'forcedownload' => true));
exit;
// TODO: delete the zipfile (and temporary files) once it's been downloaded
}
// Turn off all compression because it prevents output from being flushed
if (function_exists('apache_setenv')) {
apache_setenv('no-gzip', 1);
}
@ini_set('zlib.output_compression', 0);
if (!$exportdata = $SESSION->get('exportdata')) {
redirect(get_config('wwwroot').'admin/users/bulkexport.php');
}
$SESSION->set('exportdata', '');
$stylesheets = array_reverse($THEME->get_url('style/style.css', true));
print_export_head($stylesheets);
flush();
/**
* Outputs enough HTML to make a pretty error message in the iframe
*
* @param string $message The message to display to the user
*/
function export_iframe_die($message) {
print_export_iframe_die($message);
exit;
}
/**
* Registered as the progress report handler for the export. Streams updates
* back to the browser
*
* @param int $percent How far complete the export is
* @param string $status A human-readable string describing the current step
*/
function export_iframe_progress_handler($percent, $status) {
print_iframe_progress_handler($percent, $status);
ob_flush();
}
/**
* Convert a 2D array to a CSV file. This follows the basic rules from http://en.wikipedia.org/wiki/Comma-separated_values
*
* @param array $input 2D array of values: each line is an array of values
*/
function data_to_csv($input) {
if (empty($input) or !is_array($input)) {
return '';
}
$output = '';
foreach ($input as $line) {
$lineoutput = '';
foreach ($line as $element) {
$element = str_replace('"', '""', $element);
if (!empty($lineoutput)) {
$lineoutput .= ',';
}
$lineoutput .= "\"$element\"";
}
$output .= $lineoutput . "\r\n";
}
return $output;
}
function create_zipfile($listing, $files) {
global $USER;
if (empty($listing) or empty($files)) {
return false;
}
if (count($listing) != count($files)) {
throw new MaharaException("Files and listing don't match.");
}
// create temporary directories for the export
$exportdir = get_config('dataroot') . 'export/'
. $USER->get('id') . '/' . time() . '/';
if (!check_dir_exists($exportdir)) {
throw new SystemException("Couldn't create the temporary export directory $exportdir");
}
$usersdir = 'users/';
if (!check_dir_exists($exportdir . $usersdir)) {
throw new SystemException("Couldn't create the temporary export directory $usersdir");
}
// move user zipfiles into the export directory
foreach ($files as $filename) {
if (copy($filename, $exportdir . $usersdir . basename($filename))) {
unlink($filename);
}
else {
throw new SystemException("Couldn't move $filename to $usersdir");
}
}
// write username listing to a file
$listingfile = 'usernames.csv';
if (!file_put_contents($exportdir . $listingfile, data_to_csv($listing))) {
throw new SystemException("Couldn't write usernames to a file");
}
// zip everything up
$zipfile = $exportdir . 'mahara-bulk-export-' . time() . '.zip';
$cwd = getcwd();
$command = sprintf('%s %s %s %s %s',
get_config('pathtozip'),
get_config('ziprecursearg'),
escapeshellarg($zipfile),
escapeshellarg($listingfile),
escapeshellarg($usersdir)
);
$output = array();
chdir($exportdir);
exec($command, $output, $returnvar);
chdir($cwd);
if ($returnvar != 0) {
throw new SystemException('Failed to zip the export file: return code ' . $returnvar);
}
return $zipfile;
}
// Bail if we don't have enough data to do an export
if (empty($exportdata)) {
export_iframe_die(get_string('unabletogenerateexport', 'export'));
}
ob_start();
export_iframe_progress_handler(0, get_string('Setup', 'export'));
safe_require('export', 'leap');
$listing = array();
$files = array();
$exportcount = 0;
$exporterrors = array();
foreach ($exportdata as $username) {
$user = new User();
try {
$user->find_by_username($username);
} catch (AuthUnknownUserException $e) {
continue; // Skip non-existent users
}
$percentage = (double)$exportcount / count($exportdata) * 100;
$percentage = min($percentage, 98);
export_iframe_progress_handler($percentage, get_string('exportingusername', 'admin', $username));
$exporter = new PluginExportLeap($user, PluginExport::EXPORT_ALL_VIEWS, PluginExport::EXPORT_ALL_ARTEFACTS);
try {
$zipfile = $exporter->export();
} catch (Exception $e) {
$exporterrors[] = $username;
continue;
}
$listing[] = array($username, $zipfile);
$files[] = $exporter->get('exportdir') . $zipfile;
$exportcount++;
}
export_iframe_progress_handler(99, get_string('creatingzipfile', 'export'));
if (!$zipfile = create_zipfile($listing, $files)) {
export_iframe_die(get_string('bulkexportempty', 'admin'));
}
export_iframe_progress_handler(100, get_string('Done', 'export'));
ob_end_flush();
log_info("Exported $exportcount users to $zipfile");
if (!empty($exporterrors)) {
$SESSION->add_error_msg(get_string('couldnotexportusers', 'admin', implode(', ', $exporterrors)));
}
// Store the filename in the session, and redirect the iframe to it to trigger
// the download. Here it would be nice to trigger the download for everyone,
// but alas this is not possible for people without javascript.
$SESSION->set('exportfile', $zipfile);
$continueurljs = get_config('wwwroot');
$continueurl = 'bulkdownload.php';
$result = $SESSION->get('messages');
$SESSION->clear('messages');
print_export_footer(get_string('exportgeneratedsuccessfully1', 'export'), $continueurl, $continueurljs, $result, 'bulkdownload.php');
......@@ -16,6 +16,98 @@ require_once('pieforms/pieform.php');
define('TITLE', get_string('bulkexporttitle', 'admin'));
/**
* Convert a 2D array to a CSV file. This follows the basic rules from http://en.wikipedia.org/wiki/Comma-separated_values
*
* @param array $input 2D array of values: each line is an array of values
*/
function data_to_csv($input) {
if (empty($input) or !is_array($input)) {
return '';
}
$output = '';
foreach ($input as $line) {
$lineoutput = '';
foreach ($line as $element) {
$element = str_replace('"', '""', $element);
if (!empty($lineoutput)) {
$lineoutput .= ',';
}
$lineoutput .= "\"$element\"";
}
$output .= $lineoutput . "\r\n";
}
return $output;
}
/**
* Create a zip archive containing the exported data.
*
* @param array $listing The list of usernames that were exported
* @param array $files A list of archive files for each user
*/
function create_zipfile($listing, $files) {
global $USER;
if (empty($listing) or empty($files)) {
return false;
}
if (count($listing) != count($files)) {
throw new MaharaException("Files and listing don't match.");
}
// create temporary directories for the export
$exportdir = get_config('dataroot') . 'export/'
. $USER->get('id') . '/' . time() . '/';
if (!check_dir_exists($exportdir)) {
throw new SystemException("Couldn't create the temporary export directory $exportdir");
}
$usersdir = 'users/';
if (!check_dir_exists($exportdir . $usersdir)) {
throw new SystemException("Couldn't create the temporary export directory $usersdir");
}
// move user zipfiles into the export directory
foreach ($files as $filename) {
if (copy($filename, $exportdir . $usersdir . basename($filename))) {
unlink($filename);
}
else {
throw new SystemException("Couldn't move $filename to $usersdir");
}
}
// write username listing to a file
$listingfile = 'usernames.csv';
if (!file_put_contents($exportdir . $listingfile, data_to_csv($listing))) {
throw new SystemException("Couldn't write usernames to a file");
}
// zip everything up
$zipfile = $exportdir . 'mahara-bulk-export-' . time() . '.zip';
$cwd = getcwd();
$command = sprintf('%s %s %s %s %s',
get_config('pathtozip'),
get_config('ziprecursearg'),
escapeshellarg($zipfile),
escapeshellarg($listingfile),
escapeshellarg($usersdir)
);
$output = array();
chdir($exportdir);
exec($command, $output, $returnvar);
chdir($cwd);
if ($returnvar != 0) {
throw new SystemException('Failed to zip the export file: return code ' . $returnvar);
}
return $zipfile;
}
function bulkexport_submit(Pieform $form, $values) {
global $SESSION;
......@@ -37,12 +129,62 @@ function bulkexport_submit(Pieform $form, $values) {
}
}
$SESSION->set('exportdata', $usernames);
safe_require('export', 'leap');
$listing = array();
$files = array();
$exportcount = 0;
$exporterrors = array();
$num_users = count($usernames);
foreach ($usernames as $username) {
if (!($exportcount % 25)) {
set_progress_info('bulkexport', $exportcount, $num_users, get_string('validating', 'admin'));
}
$user = new User();
try {
$user->find_by_username($username);
}
catch (AuthUnknownUserException $e) {
continue; // Skip non-existent users
}
$exporter = new PluginExportLeap($user, PluginExport::EXPORT_ALL_VIEWS, PluginExport::EXPORT_ALL_ARTEFACTS);
try {
$zipfile = $exporter->export();
}
catch (Exception $e) {
$exporterrors[] = $username;
continue;
}
$listing[] = array($username, $zipfile);
$files[] = $exporter->get('exportdir') . $zipfile;
$exportcount++;
}
if (!$zipfile = create_zipfile($listing, $files)) {
export_iframe_die(get_string('bulkexportempty', 'admin'));
}
$smarty = smarty();
$smarty->assign('heading', '');
$smarty->display('admin/users/bulkdownload.tpl');
exit;
log_info("Exported $exportcount users to $zipfile");
if (!empty($exporterrors)) {
$SESSION->add_error_msg(get_string('couldnotexportusers', 'admin', implode(', ', $exporterrors)));
}
// Store the filename in the session, and redirect the iframe to it to trigger
// the download. Here it would be nice to trigger the download for everyone,
// but alas this is not possible for people without javascript.
$SESSION->set('exportfile', $zipfile);
set_progress_done('bulkexport', array('redirect' => '/admin/users/bulkexport.php'));
// Download the export file once it has been generated
require_once('file.php');
serve_file($zipfile, basename($zipfile), 'application/x-zip', array('lifetime' => 0, 'forcedownload' => true));
// TODO: delete the zipfile (and temporary files) once it's been downloaded
}
$authinstanceelement = array('type' => 'hidden', 'value' => '');
......@@ -76,6 +218,11 @@ $form = array(
'title' => get_string('bulkexportusernames', 'admin'),
'description' => get_string('bulkexportusernamesdescription', 'admin'),
),
'progress_meter_token' => array(
'type' => 'hidden',
'value' => 'bulkexport',
'readonly' => TRUE,
),
'submit' => array(
'type' => 'submit',
'value' => get_string('bulkexport', 'admin')
......@@ -83,6 +230,8 @@ $form = array(
)
);
set_progress_done('bulkexport');
$form = pieform($form);
$smarty = smarty();
......
......@@ -25,29 +25,6 @@ define('TITLE', get_string('bulkleap2aimport', 'admin'));
// Turn on autodetecting of line endings, so mac newlines (\r) will work
ini_set('auto_detect_line_endings', 1);
$ADDEDUSERS = $SESSION->get('bulkimport_addedusers');
if (empty($ADDEDUSERS)) {
$ADDEDUSERS = array();
}
$FAILEDUSERS = $SESSION->get('bulkimport_failedusers');
if (empty($FAILEDUSERS)) {
$FAILEDUSERS = array();
}
$LEAP2AFILES = $SESSION->get('bulkimport_leap2afiles');
if (empty($LEAP2AFILES)) {
$LEAP2AFILES = array();
}
$AUTHINSTANCE = $SESSION->get('bulkimport_authinstance');
$EMAILUSERS = $SESSION->get('bulkimport_emailusers');
// Import in progress
if (!empty($LEAP2AFILES)) {
import_next_user();
}
elseif (!empty($ADDEDUSERS) or !empty($FAILEDUSERS)) {
finish_import();
}
$authinstances = auth_get_auth_instances();
if (count($authinstances) > 0) {
......@@ -86,6 +63,11 @@ $form = array(
'description' => get_string('emailusersaboutnewaccountdescription', 'admin'),
'defaultvalue' => true,
),
'progress_meter_token' => array(
'type' => 'hidden',
'value' => 'bulkimport',
'readonly' => TRUE,
),
'submit' => array(
'type' => 'submit',
'value' => get_string('Import', 'admin')
......@@ -93,25 +75,6 @@ $form = array(
)
);
/**
* Work-around the redirection limit of Firefox (http://kb.mozillazine.org/Network.http.redirection-limit)
*/
function meta_redirect() {
global $SESSION, $LEAP2AFILES, $ADDEDUSERS, $FAILEDUSERS;
$SESSION->set('bulkimport_leap2afiles', $LEAP2AFILES);
$SESSION->set('bulkimport_addedusers', $ADDEDUSERS);
$SESSION->set('bulkimport_failedusers', $FAILEDUSERS);
$url = get_config('wwwroot') . '/admin/users/bulkimport.php';
$failed = sizeof($FAILEDUSERS) ? ' (' . sizeof($FAILEDUSERS) . ' failed)' : '';
$done = sizeof($FAILEDUSERS) + sizeof($ADDEDUSERS);
$total = $done + sizeof($LEAP2AFILES);
$title = "Completed {$done}/{$total}{$failed}";
print_meta_redirect($url, $title);
exit;
}
/**
* The CSV file is parsed here so validation errors can be returned to the
* user. The data from a successful parsing is stored in the <var>$LEAP2AFILES</var>
......@@ -196,32 +159,34 @@ function bulkimport_validate(Pieform $form, $values) {
function bulkimport_submit(Pieform $form, $values) {
global $SESSION, $LEAP2AFILES;
log_info('Attempting to import ' . count($LEAP2AFILES) . ' users from Leap2A files');
require_once('file.php');
require_once(get_config('docroot') . 'import/lib.php');
safe_require('import', 'leap');
$key = 0;
$total = count($LEAP2AFILES);
$SESSION->set('bulkimport_leap2afiles', $LEAP2AFILES);
$SESSION->set('bulkimport_authinstance', (int)$values['authinstance']);