Commit 1e3207a8 authored by Penny Leach's avatar Penny Leach
Browse files

Big refactor of importing to prepare for doing LEAP2A over mnet



This commit includes:

- Moved a lot of logic out of admin/users/add.php to the various plugins
- Moved around unzipping and detecting files into the transport layers
- Refactored a lot of transport code into the base class

I tested:

- Adding a user from a leap2a xml file
- Adding a user from a leap2a zip file
- Importing non leap (straight files) data over mnet
Signed-off-by: default avatarPenny Leach <penny@mjollnir.org>
parent 3552d005
......@@ -35,6 +35,8 @@ define('SECTION_PLUGINNAME', 'admin');
require_once('pieforms/pieform.php');
require_once('institution.php');
$TRANSPORTER = null;
if ($USER->get('admin')) {
$authinstances = auth_get_auth_instances();
}
......@@ -141,7 +143,7 @@ $form = pieform(array(
function adduser_validate(Pieform $form, $values) {
global $USER, $LEAP2A_FILE;
global $USER, $TRANSPORTER;
$authobj = AuthFactory::create($values['authinstance']);
......@@ -189,59 +191,23 @@ function adduser_validate(Pieform $form, $values) {
}
$date = time();
$nicedate = date('Y/m/d h:i:s', $date);
$niceuser = preg_replace('/[^a-zA-Z0-9_-]/', '-', $values['username']);
$uploaddir = get_config('dataroot') . 'import/' . $niceuser . '-' . $date . '/';
$filename = $uploaddir . $values['leap2afile']['name'];
check_dir_exists($uploaddir);
if (!move_uploaded_file($values['leap2afile']['tmp_name'], $filename)) {
$form->set_error('leap2afile', get_string('failedtoobtainuploadedleapfile', 'admin'));
}
if ($values['leap2afile']['type'] == 'application/octet-stream') {
// the browser wasn't sure, so use mime_content_type to guess
$mimetype = mime_content_type($filename);
}
else {
$mimetype = $values['leap2afile']['type'];
}
safe_require('artefact', 'file');
$ziptypes = PluginArtefactFile::get_mimetypes_from_description('zip');
if (in_array($mimetype, $ziptypes)) {
// Unzip the file
$command = sprintf('%s %s %s %s',
escapeshellcmd(get_config('pathtounzip')),
escapeshellarg($filename),
get_config('unzipdirarg'),
escapeshellarg($uploaddir)
);
$output = array();
exec($command, $output, $returnvar);
if ($returnvar != 0) {
log_debug("unzip command failed with return value $returnvar");
// Let's make it obvious if the cause is obvious :)
if ($returnvar == 127) {
log_debug("This means that 'unzip' isn't installed, or the config var \$cfg->pathtounzip is not"
. " pointing at unzip (see Mahara's file lib/config-defaults.php)");
}
$form->set_error('leap2afile', get_string('failedtounzipleap2afile', 'admin'));
return;
safe_require('import', 'leap');
$TRANSPORTER = new LocalImporterTransport($values['leap2afile']['tmp_name'], $values['leap2afile']['name'], $niceuser . '-' . $date);
try {
if ($values['leap2afile']['type'] == 'application/octet-stream') {
// the browser wasn't sure, so use mime_content_type to guess
$mimetype = mime_content_type($values['leap2afile']['filename']);
}
$filename = $uploaddir . 'leap2a.xml';
if (!is_file($filename)) {
$form->set_error('leap2afile', get_string('noleap2axmlfiledetected', 'admin'));
return;
else {
$mimetype = $values['leap2afile']['type'];
}
$TRANSPORTER->extract_file($mimetype);
PluginImportLeap::validate_import_data($TRANSPORTER->files_info());
}
else if ($mimetype != 'text/xml') {
$form->set_error('leap2afile', get_string('fileisnotaziporxmlfile', 'admin'));
catch (Exception $e) {
$form->set_error('leap2afile', $e->getMessage());
}
$LEAP2A_FILE = $filename;
}
else {
if (!$form->get_error('firstname') && !preg_match('/\S/', $firstname)) {
......@@ -266,7 +232,7 @@ function adduser_validate(Pieform $form, $values) {
}
function adduser_submit(Pieform $form, $values) {
global $USER, $SESSION, $LEAP2A_FILE;
global $USER, $SESSION, $TRANSPORTER;
db_begin();
ini_set('max_execution_time', 180);
......@@ -305,28 +271,22 @@ function adduser_submit(Pieform $form, $values) {
if (isset($values['leap2afile'])) {
// And we're good to go
$filename = substr($LEAP2A_FILE, strlen(get_config('dataroot')));
$logfile = dirname($LEAP2A_FILE) . '/import.log';
require_once(get_config('docroot') . 'import/lib.php');
safe_require('import', 'leap');
$importer = PluginImport::create_importer(null, (object)array(
$importdata = (object)array(
'token' => '',
//'host' => '',
'usr' => $user->id,
'queue' => (int)!(PluginImport::import_immediately_allowed()), // import allowed straight away? Then don't queue
'ready' => 0, // maybe 1?
'expirytime' => db_format_timestamp(time()+(60*60*24)),
'format' => 'leap',
'data' => array('filename' => $filename),
'loglevel' => PluginImportLeap::LOG_LEVEL_VERBOSE,
'logtargets' => LOG_TARGET_FILE,
'logfile' => $logfile,
'profile' => true,
));
);
$importer = PluginImport::create_importer(null, $TRANSPORTER, $importdata);
try {
$importer->process();
log_info("Imported user account $user->id from leap2a file, see $logfile for a full log");
log_info("Imported user account $user->id from leap2a file, see " . $importer->get('logfile') . ' for a full log');
}
catch (ImportException $e) {
log_info("LEAP2A import failed: " . $e->getMessage());
......
......@@ -350,7 +350,8 @@ function send_content_ready($token, $username, $format, $importdata, $fetchnow=f
$result = new StdClass;
if ($fetchnow && PluginImport::import_immediately_allowed()) {
// either immediately spawn a curl request to go fetch the file
$importer = PluginImport::create_importer($queue->id, $queue);
$tr = new MnetImporterTransport($queue);
$importer = PluginImport::create_importer($queue->id, $tr, $queue);
$importer->prepare();
$importer->process();
$importer->cleanup();
......
......@@ -31,7 +31,6 @@ class PluginImportFile extends PluginImport {
private $manifest;
private $files;
private $unzipdir;
private $zipfilesha1;
private $artefacts;
private $importdir;
......@@ -59,42 +58,14 @@ class PluginImportFile extends PluginImport {
}
public function process() {
$this->extract_file();
$this->importertransport->extract_file($this->importertransport->get('mimetype'), $this->zipfilesha1);
$this->verify_file_contents();
$this->add_artefacts();
}
public function extract_file() {
$filesinfo = $this->get('importertransport')->files_info();
// this contains relativepath and zipfile name
$this->relativepath = $filesinfo['relativepath'];
$this->zipfile = $filesinfo['zipfile'];
$this->tempdir = $filesinfo['tempdir'];
if (sha1_file($this->tempdir . $this->zipfile) != $this->zipfilesha1) {
throw new ImportException($this, 'sha1 of recieved zipfile didn\'t match expected sha1');
}
$this->unzipdir = $this->tempdir . 'extract/';
if (!check_dir_exists($this->unzipdir)) {
throw new ImportException($this, 'Failed to create the temporary directories to work in');
}
$command = sprintf('%s %s %s %s',
get_config('pathtounzip'),
escapeshellarg($this->tempdir . $this->zipfile),
get_config('unzipdirarg'),
escapeshellarg($this->unzipdir)
);
$output = array();
exec($command, $output, $returnvar);
if ($returnvar != 0) {
throw new ImportException($this, 'Failed to unzip the file recieved from the transport object');
}
}
public function verify_file_contents() {
$includedfiles = get_dir_contents($this->unzipdir);
$uzd = $this->importertransport->get('tempdir') . 'extract/';
$includedfiles = get_dir_contents($uzd);
$okfiles = array();
$badfiles = array();
// check what arrived in the directory
......@@ -105,7 +76,7 @@ class PluginImportFile extends PluginImport {
unset($includedfiles[$k]);
continue;
}
$sha1 = sha1_file($this->unzipdir . $f);
$sha1 = sha1_file($uzd . $f);
if (array_key_exists($sha1, $this->manifest)) {
$tmp = new StdClass;
$tmp->sha1 = $sha1;
......@@ -131,6 +102,7 @@ class PluginImportFile extends PluginImport {
public function add_artefacts() {
// we're just adding them as files into an 'incoming' directory in the user's file area.
safe_require('artefact', 'file');
$uzd = $this->importertransport->get('tempdir') . 'extract/';
try {
$this->importdir = ArtefactTypeFolder::get_folder_id('incoming', get_string('incomingfolderdesc'), null, true, $this->get('usr'));
} catch (Exception $e) {
......@@ -148,13 +120,13 @@ class PluginImportFile extends PluginImport {
'locked' => 0,
);
if ($imagesize = getimagesize($this->tempdir . 'extract/' . $f->actualfilename)) {
if ($imagesize = getimagesize($uzd . $f->actualfilename)) {
$mime = $imagesize['mime'];
$data->filetype = $mime;
}
$id = ArtefactTypeFile::save_file(
$this->tempdir . 'extract/' . $f->actualfilename,
$uzd . $f->actualfilename,
$data,
$this->get('usrobj'),
true
......
......@@ -67,6 +67,23 @@ class PluginImportLeap extends PluginImport {
const STRATEGY_IMPORT_AS_VIEW = 1;
public static function validate_import_data($importdata) {
if (!$file = self::find_file($importdata)) {
throw new ImportException(null, 'Missing leap xml file');
}
}
public static function find_file($importdata) {
$path = $importdata['tempdir'] . 'extract/';
if (!empty($importdata['manifestfile'])) {
$files = array($importdata['manifestfile']);
} else {
$files = array('leap.xml', 'leap2.xml', 'leap2a.xml');
}
foreach ($files as $f) {
if (file_exists($path . $f)) {
return $path . $f;
}
}
}
public function get($field) {
......@@ -79,8 +96,8 @@ class PluginImportLeap extends PluginImport {
public function process() {
db_begin();
$data = $this->get('data');
$filename = get_config('dataroot') . $data['filename'];
$filename = self::find_file($this->get('importertransport')->files_info());
$this->logfile = dirname($filename) . '/import.log';
$this->trace('Loading import from ' . $filename);
$this->snapshot('begin');
......
......@@ -27,17 +27,25 @@
defined('INTERNAL') || die();
/**
* base class for imports.
* handles queuing and sets up some basic helper functions
*/
abstract class PluginImport extends Plugin {
private $id;
private $data;
private $host; // this might move
private $expirytime;
private $token;
private $usr;
private $usrobj;
private $importertransport;
protected $id;
protected $data;
protected $expirytime;
protected $usr;
protected $usrobj;
/** the ImporterTransport object to use */
protected $importertransport;
/**
* @param int $id the queue record id
* @param stdclass $record (optional, pass this to save db queries)
*/
public function __construct($id, $record=null) {
if (empty($record)) {
if (!$record = get_record('import_queue', 'id', $id)) {
......@@ -52,29 +60,40 @@ abstract class PluginImport extends Plugin {
}
$this->usrobj = new User();
$this->usrobj->find_by_id($this->usr);
}
if (!empty($this->host)) {
$this->importertransport = new MnetImporterTransport($this);
}
else {
$this->importertransport = new LocalImporterTransport($this);
}
// we could do more here later I guess
/**
* set the importer transport to use for this import
*
* @param ImporterTransport $transport
*/
public function set_transport(ImporterTransport $transport) {
$this->importertransport = $transport;
}
/**
* initialisation. by default just calls the transporter's prepare method
*/
public function prepare() {
$this->importertransport->prepare_files();
}
/**
* processes the files and adds them to the user's artefact area
* process the files and adds them to the user's artefact area
*/
public abstract function process();
/**
* perform cleanup tasks, delete temp files etc
*/
public function cleanup() {
$this->importertransport->cleanup();
}
/**
* helper method to return member variables
* @todo maybe refactor this to just use __get
*/
public function get($field) {
if (!property_exists($this,$field)) {
throw new ParamOutOfRangeException("Field $field wasn't found in class " . get_class($this));
......@@ -82,10 +101,22 @@ abstract class PluginImport extends Plugin {
return $this->{$field};
}
/**
* helper function to return the appropriate class name from an import format
* this will try and resolve inconsistencies (eg file/files, leap/leap2a etc
* and also pull in the class definition for you
*/
public static function class_from_format($format) {
$format = trim($format);
if ($format == 'files') {
$format = 'file';
$corr = array(
'files' => 'file',
'leap2a' => 'leap'
);
foreach ($corr as $bad => $good) {
if ($format == $bad) {
$format = $good;
break;
}
}
safe_require('import', $format);
return generate_class_name('import', $format);
......@@ -116,20 +147,42 @@ abstract class PluginImport extends Plugin {
return $queue;
}
public static function create_importer($id, $record=null) {
/**
* creates an importer object from the queue information
*
* @param int $id the queue record (if there is one, else pass 0)
* @param ImporterTransport $transport the transporter object to use
* @param stdclass $record the queue data (this <b>must</b> be passed when no id is given
*
* @return PluginImport
*/
public static function create_importer($id, ImporterTransport $transporter, $record=null) {
if (empty($record)) {
if (!$record = get_record('import_queue', 'id', $id)) {
throw new NotFoundException("Failed to find import queue record with id $id");
}
}
$class = self::class_from_format($record->format);
return new $class($id,$record);
$i = new $class($id,$record);
$i->set_transport($transporter);
$transporter->set_importer($i);
return $i;
}
/**
* validate the import data (usually what files_info returns
* @throws ImportException
*/
public static abstract function validate_import_data($importdata);
/**
* Whether imports are allowed immediately or if they must be queued
* eg if the server is under load or whatever
* @todo not implemented yet, but <b>use this anyway</b>
*
* @return boolean
*/
public static final function import_immediately_allowed() {
// @todo change this (check whatever)
return true;
}
......@@ -144,6 +197,9 @@ abstract class PluginImport extends Plugin {
}
}
/**
* cron job to process the queue and wake up and finish imports
*/
function import_process_queue() {
if (!$ready = get_records_select_array('import_queue',
......@@ -161,7 +217,14 @@ function import_process_queue() {
$processed[] = $item->id;
continue;
}
$importer = PluginImport::create_importer($item->id, $item);
$tr = null;
if (!empty($item->host)) {
$tr = new MnetImporterTransport($item);
}
else {
$tr = new LocalImporterTransport($item);
}
$importer = PluginImport::create_importer($item->id, $tr, $item);
try {
$importer->prepare();
$importer->process();
......@@ -185,14 +248,82 @@ function import_process_queue() {
);
}
/**
* base class for transport layers.
* Implements helper methods and makes some abstract stuff
*/
abstract class ImporterTransport {
/** temporary directory to work in if necessary */
protected $tempdir;
/** the importer to eventually handle the import */
protected $importer;
/** unique id for the import directories. usually the import queue id, but sometimes needs to be set manually */
protected $importid;
/** relative path inside the temporary directory */
protected $relativepath;
/** whether the tempdir has been set up already */
private $tempdirprepared = false;
/** the file to import (sometimes a zip file) */
protected $importfile;
/** the manifest file, if there is one and we know about it */
protected $manifestfile;
/** the mimetype of the file we are importing */
protected $mimetype;
/**
* figure out the temporary directory to use
* and make sure it exists, etc
*/
public function prepare_tempdir() {
if ($this->tempdirprepared) {
return true;
}
$this->relativepath = 'temp/import/' . $this->importid . '/';
if ($tmpdir = get_config('unziptempdir')) {
$this->tempdir = $tmpdir . $this->relativepath;
}
else {
$this->tempdir = get_config('dataroot') . $this->relativepath;
}
if (!check_dir_exists($this->tempdir)) {
throw new ImportException($this->importer, 'Failed to create the temporary directories to work in');
}
$this->tempdirprepared = true;
}
/**
* helper get method
* @todo maybe refactor this to __get
*/
public function get($field) {
if (!property_exists($this,$field)) {
throw new ParamOutOfRangeException("Field $field wasn't found in class " . get_class($this));
}
return $this->{$field};
}
/**
* this might be a path to a directory containing the files
* or an array containing some other info
* or the path to a file, depending on the format
*/
public abstract function files_info();
public function files_info() {
return array(
'importfile' => $this->importfile,
'tempdir' => $this->tempdir,
'relativepath' => $this->relativepath,
'manifestfile' => $this->manifestfile,
);
}
/**
* do whatever is necessary to retrieve the file(s)
......@@ -202,64 +333,115 @@ abstract class ImporterTransport {
/**
* cleanup temporary working area
*/
public abstract function cleanup();
}
class LocalImporterTransport extends ImporterTransport {
private $relativepath;
private $zipfilename;
public function __construct(PluginImport $importer) {
}
public function cleanup() {
// TODO
if (empty($this->tempdir)) {
return;
}
require_once('file.php');
rmdirr($this->tempdir);
}
public function prepare_files() {
/*
* set the importer object
* this must be done before prepare_files is called
*
* @param PluginImport $importer
*/
public function set_importer(PluginImport $importer) {
$this->importer = $importer;
}
/**
* For this to work with the 'file' import plugin, it needs to provide 'zipfile' and 'relativepath'
* helper function for import code to use to extract a file
* it will either unzip a zip file, or move an import file to the destination
*
* Other import plugins might need different things
* @param string $expectedsha1 optional, if given will validate the sha1 first
*
* @throws ImportException
*/
public function files_info() {
return array(
'zipfile' => $this->zipfilename,
'relativepath' => $this->relativepath,
public function extract_file($mimetype, $expectedsha1=null) {
$this->prepare_tempdir();
if ($expectedsha1 && sha1_file($this->importfile) != $expectedsha1) {
throw new ImportException($this->importer, 'sha1 of recieved importfile didn\'t match expected sha1');
}
$todir = $this->tempdir . 'extract/';
if (!check_dir_exists($todir)) {
throw new ImportException($this, 'Failed to create the temporary directories to work in');
}
safe_require('artefact', 'file');
$ziptypes = PluginArtefactFile::get_mimetypes_from_description('zip');
// if we don't have a zipfile, just move the import file to the extract location
if (!in_array($mimetype, $ziptypes)) {
if (strpos($this->importfile, $todir) !== 0) {
rename($this->importfile, $todir . $this->importfilename);
}
$this->manifestfile = $this->importfilename;
return;
}
$command = sprintf('%s %s %s %s',
get_config('pathtounzip'),
escapeshellarg($this->importfile),
get_config('unzipdirarg'),
escapeshellarg($todir)
);
$output = array();
exec($command, $output, $returnvar);
if ($returnvar != 0) {
throw new ImportException($this, 'Failed to unzip the file recieved from the transport object');
}
}
}
/**
* class to handle 'local' transport - eg uploaded files
*/
class LocalImporterTransport extends ImporterTransport {
class MnetImporterTransport extends ImporterTransport {
public function __construct($importfile, $importfilename, $importid) {
$this->importfile = $importfile;
$this->importfilename = $importfilename;
$this->importid = $importid;
}