Commit d022f93a authored by Hugh Davenport's avatar Hugh Davenport
Browse files

Add page for Group CSV Uploads



You can now use the web API service for groups to upload
a CSV file to create and update groups

The new page is below, and gives all the details about the format
/admin/groups/uploadcsv.php

Bug #547688

Change-Id: Ic116c836e607ae3d1f6c0f67bbbf1f9c119828a8
Signed-off-by: default avatarHugh Davenport <hugh@catalyst.net.nz>
parent 106152de
<?php
/**
* Mahara: Electronic portfolio, weblog, resume builder and social networking
* Copyright (C) 2011 Catalyst IT Ltd and others; see:
* http://wiki.mahara.org/Contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package mahara
* @subpackage admin
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL
* @copyright (C) 2011 Catalyst IT Ltd http://catalyst.net.nz
*
*/
define('INTERNAL', 1);
define('INSTITUTIONALADMIN', 1);
define('MENUITEM', 'managegroups/uploadcsv');
require(dirname(dirname(dirname(__FILE__))) . '/init.php');
define('TITLE', get_string('uploadgroupcsv', 'admin'));
require_once('pieforms/pieform.php');
require_once(get_config('libroot') . 'group.php');
require_once(get_config('libroot') . 'institution.php');
safe_require('artefact', 'internal');
raise_memory_limit("512M");
// Turn on autodetecting of line endings, so mac newlines (\r) will work
ini_set('auto_detect_line_endings', 1);
$FORMAT = array();
$ALLOWEDKEYS = array(
'shortname',
'displayname',
'description',
'grouptype',
'public',
);
if ($USER->get('admin')) {
$ALLOWEDKEYS[] = 'usersautoadded';
$ALLOWEDKEYS[] = 'quota';
}
$MANDATORYFIELDS = array(
'shortname',
'displayname',
'grouptype',
);
$UPDATES = array(); // During validation, remember which group already exist
$JOINTYPES = array();
foreach (group_get_grouptypes() as $type) {
safe_require('grouptype', $type);
$JOINTYPES[$type] = call_static_method('GroupType' . $type, 'allowed_join_types');
}
$form = array(
'name' => 'uploadcsv',
'elements' => array(
'institution' => get_institution_selector(),
'file' => array(
'type' => 'file',
'title' => get_string('csvfile', 'admin'),
'description' => get_string('groupcsvfiledescription', 'admin'),
'rules' => array(
'required' => true
)
),
'updategroups' => array(
'type' => 'checkbox',
'title' => get_string('updategroups', 'admin'),
'description' => get_string('updategroupsdescription', 'admin'),
'defaultvalue' => false,
),
'submit' => array(
'type' => 'submit',
'value' => get_string('uploadgroupcsv', 'admin')
)
)
);
/**
* The CSV file is parsed here so validation errors can be returned to the
* user. The data from a successful parsing is stored in the <var>$CVSDATA</var>
* array so it can be accessed by the submit function
*
* @param Pieform $form The form to validate
* @param array $values The values submitted
*/
function uploadcsv_validate(Pieform $form, $values) {
global $CSVDATA, $ALLOWEDKEYS, $MANDATORYFIELDS, $JOINTYPES, $FORMAT, $USER, $UPDATES;
// Don't even start attempting to parse if there are previous errors
if ($form->has_errors()) {
return;
}
if ($values['file']['size'] == 0) {
$form->set_error('file', $form->i18n('rule', 'required', 'required', array()));
return;
}
$institution = $values['institution'];
if (!$USER->can_edit_institution($institution)) {
$form->set_error('institution', get_string('notadminforinstitution', 'admin'));
return;
}
require_once('csvfile.php');
$csvgroups = new CsvFile($values['file']['tmp_name']);
$csvgroups->set('allowedkeys', $ALLOWEDKEYS);
$csvgroups->set('mandatoryfields', $MANDATORYFIELDS);
$csvdata = $csvgroups->get_data();
if (!empty($csvdata->errors['file'])) {
$form->set_error('file', $csvdata->errors['file']);
return;
}
$csverrors = new CSVErrors();
$formatkeylookup = array_flip($csvdata->format);
$shortnames = array();
$displaynames = array();
foreach ($csvdata->data as $key => $line) {
// If headers exists, increment i = key + 2 for actual line number
$i = ($csvgroups->get('headerExists')) ? ($key + 2) : ($key + 1);
// Trim non-breaking spaces -- they get left in place by File_CSV
foreach ($line as &$field) {
$field = preg_replace('/^(\s|\xc2\xa0)*(.*?)(\s|\xc2\xa0)*$/', '$2', $field);
}
if (count($line) != count($csvdata->format)) {
$csverrors->add($i, get_string('uploadcsverrorwrongnumberoffields', 'admin', $i));
continue;
}
$shortname = $line[$formatkeylookup['shortname']];
$displayname = $line[$formatkeylookup['displayname']];
$grouptype = $line[$formatkeylookup['grouptype']];
if (!preg_match('/^[a-zA-Z0-9_.-]{2,255}$/', $shortname)) {
$csverrors->add($i, get_string('uploadgroupcsverrorinvalidshortname', 'admin', $i, $shortname));
}
if (isset($shortnames[$shortname])) {
// Duplicate shortname within this file.
$csverrors->add($i, get_string('uploadgroupcsverrorshortnamealreadytaken', 'admin', $i, $shortname));
}
else if (!$values['updategroups']) {
// The groupname must be new
if (record_exists('group', 'shortname', $shortname, 'institution', $institution)) {
$csverrors->add($i, get_string('uploadgroupcsverrorshortnamealreadytaken', 'admin', $i, $shortname));
}
}
$shortnames[$shortname] = array(
'shortname' => $shortname,
'displayname' => $displayname,
'grouptype' => $grouptype,
'lineno' => $i,
'raw' => $line,
);
if (isset($displaynames[strtolower($displayname)])) {
// Duplicate displayname within this file
$csverrors->add($i, get_string('uploadgroupcsverrorsgroupnamealreadyexists', 'admin', $i, $displayname));
}
else if (!$values['updategroups']) {
// The displayname must be new
if (get_records_sql_array('SELECT id FROM {group} WHERE LOWER(TRIM(name)) = ?', array(strtolower(trim($displayname))))) {
$csverrors->add($i, get_string('uploadgroupcsverrorgroupnamealreadyexists', 'admin', $i, $displayname));
}
}
else {
// This displayname must be new if not our shortname/institution
if (get_records_sql_array('
SELECT id FROM {group}
WHERE LOWER(TRIM(name)) = ?
AND NOT (shortname = ? AND institution = ?)',
array(
strtolower(trim($displayname)),
$shortname,
$institution
))) {
$csverrors->add($i, get_string('uploadgroupcsverrorgroupnamealreadyexists', 'admin', $i, $displayname));
}
}
$displaynames[strtolower($displayname)] = 1;
$groupjointype = split('/', $grouptype);
if (count($groupjointype) != 2) {
$csverrors->add($i, get_string('uploadgroupcsverrorinvalidgrouptype', 'admin', $i, $grouptype));
}
else {
if (!isset($JOINTYPES[$groupjointype[0]]) || !in_array($groupjointype[1], $JOINTYPES[$groupjointype[0]])) {
$csverrors->add($i, get_string('uploadgroupcsverrorinvalidgrouptype', 'admin', $i, $grouptype));
}
}
if ($values['updategroups']) {
foreach ($shortnames as $shortname => $data) {
// TODO: Any other checks we have to do for updated groups
$UPDATES[$shortname] = 1;
}
}
}
if ($errors = $csverrors->process()) {
$form->set_error('file', clean_html($errors));
return;
}
$FORMAT = $csvdata->format;
$CSVDATA = $csvdata->data;
}
/**
* Add the users to the system. Make sure that they have to change their
* password on next login also.
*/
function uploadcsv_submit(Pieform $form, $values) {
global $SESSION, $CSVDATA, $FORMAT, $UPDATES, $USER;
$formatkeylookup = array_flip($FORMAT);
$institution = $values['institution'];
if ($values['updategroups']) {
log_info('Updating groups from the CSV file');
}
else {
log_info('Inserting groups from the CSV file');
}
db_begin();
$addedgroups = array();
foreach ($CSVDATA as $record) {
$groupjointype = split('/', $record[$formatkeylookup['grouptype']]);
$group = new StdClass;
$group->name = $record[$formatkeylookup['displayname']];
$group->shortname = $record[$formatkeylookup['shortname']];
$group->institution = $institution;
$group->grouptype = $groupjointype[0];
$group->jointype = $groupjointype[1];
foreach ($FORMAT as $field) {
if ($field == 'displayname' || $field == 'shortname' || $field == 'grouptype') {
continue;
}
$group->{$field} = $record[$formatkeylookup[$field]];
}
if (!$values['updategroups'] || !isset($UPDATES[$group->shortname])) {
$group->members = array($USER->id => 'admin');
$group->id = group_create((array)$group);
$addedgroups[] = $group;
log_debug('added group ' . $group->name);
}
else if (isset($UPDATES[$group->shortname])) {
$shortname = $group->shortname;
$updates = group_update($group);
if (empty($updates)) {
unset($UPDATES[$shortname]);
}
else {
if (isset($updates['name'])) {
$updates['displayname'] = $updates['name'];
unset($updates['name']);
}
$UPDATES[$shortname] = $updates;
log_debug('updated group ' . $group->name . ' (' . implode(', ', array_keys((array)$updates)) . ')');
}
}
}
db_commit();
$SESSION->add_ok_msg(get_string('csvfileprocessedsuccessfully', 'admin'));
if ($UPDATES) {
$updatemsg = smarty_core();
$updatemsg->assign('added', count($addedgroups));
$updatemsg->assign('updates', $UPDATES);
$SESSION->add_info_msg($updatemsg->fetch('admin/groups/csvupdatemessage.tpl'), false);
}
else {
$SESSION->add_ok_msg(get_string('numbernewgroupsadded', 'admin', count($addedgroups)));
}
redirect('/admin/groups/uploadcsv.php');
}
$grouptypes = "<ul class=fieldslist>\n";
foreach ($JOINTYPES as $grouptype => $jointypes) {
foreach ($jointypes as $type) {
$grouptypes .= '<li>' . hsc($grouptype) . '/' . hsc($type) . "</li>\n";
}
}
$grouptypes .= "<div class=cl></div></ul>\n";
$fields = "<ul class=fieldslist>\n";
foreach ($ALLOWEDKEYS as $type) {
$helplink = '';
if ($type == 'grouptype' || $type == 'public' || $type == 'usersautoadded') {
$helplink = get_help_icon('core', 'groups', 'creategroup', $type);
}
$fields .= '<li>' . hsc($type) . $helplink . "</li>\n";
}
$fields .= "<div class=cl></div></ul>\n";
$uploadcsvpagedescription = get_string('uploadgroupcsvpagedescription2', 'admin', get_help_icon('core', 'groups', 'creategroup', 'grouptype'), $grouptypes, $fields);
$form = pieform($form);
$smarty = smarty(array('adminuploadcsv'));
$smarty->assign('uploadcsvpagedescription', $uploadcsvpagedescription);
$smarty->assign('uploadcsvform', $form);
$smarty->assign('PAGEHEADING', TITLE);
$smarty->display('admin/groups/uploadcsv.tpl');
......@@ -100,6 +100,7 @@ $string['adminnotifications'] = 'Admin Notifications';
$string['adminnotificationsdescription'] = 'Overview of how administrators receive system notifications';
$string['uploadcsv'] = 'Add Users by CSV';
$string['uploadcsvdescription'] = 'Upload a CSV file containing new users';
$string['uploadgroupcsv'] = 'Add Groups by CSV';
$string['usersearch'] = 'User Search';
$string['usersearchdescription'] = 'Search all users and perform administrative actions on them';
$string['usersearchinstructions'] = 'You can search for users by clicking on the initials of their first and last names, or by entering a name in the search box. You can also enter an email address in the search box if you would like to search email addresses.';
......@@ -107,6 +108,7 @@ $string['usersearchinstructions'] = 'You can search for users by clicking on the
$string['administergroups'] = 'Administer Groups';
$string['administergroupsdescription'] = 'Appoint group administrators and delete groups';
$string['groupcategoriesdescription'] = 'Add and edit group categories';
$string['uploadgroupcsvdescription'] = 'Upload a CSV file containing new groups';
$string['institutionmembersdescription'] = 'Associate users with institutions';
$string['institutionstaffdescription'] = 'Assign users Staff permissions';
......@@ -410,6 +412,7 @@ $string['forceuserstochangepassworddescription'] = 'Whether users should be forc
$string['uploadcsvinstitution'] = 'The institution and authentication method for the new users';
$string['configureauthplugin'] = 'You must configure an authentication plugin before you can add users';
$string['csvfiledescription'] = 'The file containing users to add';
$string['groupcsvfiledescription'] = 'The file containing groups to add';
$string['csverroremptyfile'] = 'The csv file is empty.';
$string['invalidfilename'] = 'The file "%s" does not exist';
$string['uploadcsverrorinvalidfieldname'] = 'The field name "%s" is invalid, or you have more fields than your header row specifies';
......@@ -448,15 +451,40 @@ $string['uploadcsvpagedescription2institutionaladmin'] = '<p>You may use this fa
<p>Your CSV file may include any other profile fields as you require. The full list of fields is:</p>
%s';
$string['uploadgroupcsverrorgroupnamealreadyexists'] = 'Error on line %s of your file: The groupname "%s" already exists';
$string['uploadgroupcsverrorinvalidshortname'] = 'Error on line %s of your file: The shortname "%s" is invalid';
$string['uploadgroupcsverrorinvalidgrouptype'] = 'Error on line %s of your file: The grouptype "%s" is invalid';
$string['uploadgroupcsverrorshortnamealreadytaken'] = 'Error on line %s of your file: The shortname "%s" is already taken';
$string['uploadgroupcsverrorusernamesnotlastfield'] = 'The "usernames" field must be the last field in the header';
$string['uploadgroupcsvpagedescription2'] = '<p>You may use this facility to upload new groups via a <acronym title="Comma Separated Values">CSV</acronym> file.</p>
<p>The first row of your CSV file should specify the format of your CSV data. For example, it should look like this:</p>
<pre>shortname,displayname,grouptype</pre>
<p>This row must include the <tt>shortname</tt>, <tt>displayname</tt>, and <tt>grouptype</tt> fields</p>
<p>The grouptype field can have any of the following: %s</p>
%s
<p>Your CSV file may include any other fields as you require. The full list of fields is:</p>
%s';
$string['uploadcsvsomeuserscouldnotbeemailed'] = 'Some users could not be e-mailed. Their e-mail addresses may be invalid, or the server Mahara is running on might not be configured to send e-mail properly. The server error log has more details. For now, you may want to contact these people manually:';
$string['uploadcsvfailedusersexceedmaxallowed'] = 'No users have been added because there are too many users in your file. The number of users in the institution would have exceeded the maximum number allowed.';
$string['updateusers'] = 'Update Users';
$string['updateusersdescription'] = 'If your CSV file contains the usernames of users who are already members of the institution you have specified, their details will be overwritten with data from the file. Use with care.';
$string['updategroups'] = 'Update Groups';
$string['updategroupsdescription'] = 'If your CSV file contains the group shortname of groups who are already in the Mahara system, their details will be overwritten with data from the file. Use with care.';
$string['csvfileprocessedsuccessfully'] = 'Your CSV file was processed successfully';
$string['nousersadded'] = 'No users were added.';
$string['nogroupsadded'] = 'No groups were added.';
$string['numbernewusersadded'] = 'New users added: %s.';
$string['numbernewgroupsadded'] = 'New groups added: %s.';
$string['numberusersupdated'] = 'Users updated: %d.';
$string['numbergroupsupdated'] = 'Groups updated: %d.';
$string['showupdatedetails'] = 'Show update details';
// Bulk Leap2A import
......
......@@ -424,8 +424,18 @@ function group_update($new, $create=false) {
}
}
$diff = array_diff_assoc((array)$new, (array)$old);
if (empty($diff)) {
return null;
}
db_begin();
if (isset($new->members)) {
group_update_members($new->id, $new->members);
unset($new->members);
}
update_record('group', $new, 'id');
// When jointype changes from invite/request to anything else,
......@@ -488,6 +498,7 @@ function group_update($new, $create=false) {
db_commit();
return $diff;
}
......@@ -666,8 +677,11 @@ function group_update_members($groupid, $members) {
WHERE usr IN (' . join(',', $userids) . ') AND institution = ?',
array($group->institution)
);
if ($baduserids = array_diff($userids, $gooduserids)) {
throw new UserException("group_update_members: some members are not in the institution $group->institution: " . join(',', $baduserids));
if (!(count($baduserids) == 1 && $baduserids[0] == $USER->id)) {
throw new UserException("group_update_members: some members are not in the institution $group->institution: " . join(',', $baduserids));
}
}
}
else {
......
......@@ -1789,6 +1789,12 @@ function admin_nav() {
'title' => get_string('groupcategories', 'admin'),
'weight' => 20,
),
'managegroups/uploadcsv' => array(
'path' => 'managegroups/uploadcsv',
'url' => 'admin/groups/uploadcsv.php',
'title' => get_string('uploadgroupcsv', 'admin'),
'weight' => 30,
),
'manageinstitutions' => array(
'path' => 'manageinstitutions',
'url' => 'admin/users/institutions.php',
......@@ -1904,11 +1910,23 @@ function institutional_admin_nav() {
'title' => get_string('uploadcsv', 'admin'),
'weight' => 40,
),
'managegroups' => array(
'path' => 'managegroups',
'url' => 'admin/groups/uploadcsv.php',
'title' => get_string('managegroups', 'admin'),
'weight' => 20,
),
'managegroups/uploadcsv' => array(
'path' => 'managegroups/uploadcsv',
'url' => 'admin/groups/uploadcsv.php',
'title' => get_string('uploadgroupcsv', 'admin'),
'weight' => 10,
),
'manageinstitutions' => array(
'path' => 'manageinstitutions',
'url' => 'admin/users/institutions.php',
'title' => get_string('manageinstitutions', 'admin'),
'weight' => 20,
'weight' => 30,
),
'manageinstitutions/institutions' => array(
'path' => 'manageinstitutions/institutions',
......
{if $added}{str tag=numbernewgroupsadded section=admin arg1=$added}{else}{str tag=nogroupsadded section=admin}{/if}
{str tag=numbergroupsupdated section=admin arg1=count($updates)}
<a href="" onclick="toggleElementClass('hidden', 'csvupdateinfo'); return false;">{str tag=showupdatedetails section=admin}</a>
<div id="csvupdateinfo" class="hidden">
{foreach from=$updates key=shortname item=fields}{strip}
<div>&nbsp;{$shortname}:&nbsp;
{foreach from=$fields key=k item=v name=fields}
{$k} &rarr; {$v}{if !$dwoo.foreach.fields.last},&nbsp;{/if}
{/foreach}
</div>{/strip}
{/foreach}
</div>
{include file="header.tpl"}
{$uploadcsvpagedescription|safe}
{$uploadcsvform|safe}
{include file="footer.tpl"}
......@@ -99,6 +99,7 @@
<ul>
<li><strong><a href="{$WWWROOT}admin/groups/groups.php">{str tag=administergroups section=admin}</a></strong> - {str tag=administergroupsdescription section=admin}</li>
<li><strong><a href="{$WWWROOT}admin/groups/groupcategories.php">{str tag=groupcategories section=admin}</a></strong> - {str tag=groupcategoriesdescription section=admin}</li>
<li><strong><a href="{$WWWROOT}admin/groups/uploadcsv.php">{str tag=uploadgroupcsv section=admin}</a></strong> - {str tag=uploadgroupcsvdescription section=admin}</li>
</ul>
<h3>{str tag=manageinstitutions section=admin}</h3>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment