Commit 9dda41d0 authored by Penny Leach's avatar Penny Leach
Browse files

ROUGH AS GUTS first commit of portfolio importer

still to do:

- cron processing is completely untested
- i would like to split out the import table based on the transport:
  import_queue.host and token should go into a import_queue_mnet table
- it's possibly worth thinking about making import a proper plugin type.
  not sure about the effect this has on the import transport
  framework...  it might be possible to have both import and
  import transport plugintypes but that might be too heavy
- at the very least if we split out import_queue.host and token into an
  mnet table it will pave the way for a better refactor laterz.
- i would still really like to improve the mnet namespacing but that
  might be plausible at this point.
- need to write docs about arguments and return types
- i want to change the content_ready arguments to not include
  $filesmanifest as that is dependent on format being file - it may
  be actually better to dispatch somewhere else based on $format and
  then just have a generic $data which would be $filesmanifest for files
  and then something else for something like LEAP or maharanative or
  whatever, as this is checked in the importer, not not the
  importertransport.
parent 5e4b2c79
......@@ -52,7 +52,16 @@ class Dispatcher {
'auth/mnet/auth.php/keepalive_server' => 'xmlrpc_not_implemented',
'auth/mnet/auth.php/kill_children' => 'xmlrpc_not_implemented',
'auth/mnet/auth.php/kill_child' => 'xmlrpc_not_implemented',
)
),
'portfolio_in' => array(
'portfolio/mahara/lib.php/send_content_intent' => 'send_content_intent',
'portfolio/mahara/lib.php/send_content_ready' => 'send_content_ready',
),
/* later...
'portfolio_out' => array(
),
*/
);
private $methodhelp = array(
......@@ -84,7 +93,14 @@ class Dispatcher {
'description' => 'username - The id of the user.'
)
)
)
),
'send_content_intent' => array(
array(
array('type' => 'string',
'description' => 'The username of the user on the remote system (previously sent in jump/land request)'
),
)
),
);
function __construct($payload, $payload_signed, $payload_encrypted) {
......
......@@ -121,10 +121,8 @@ function api_dummy_method($methodname, $argsarray, $functionname) {
return call_user_func_array($functionname, $argsarray);
}
function fetch_user_image($username) {
global $REMOTEWWWROOT;
$institution = get_field('host', 'institution', 'wwwroot', $REMOTEWWWROOT);
function find_remote_user($username, $wwwroot) {
$institution = get_field('host', 'institution', 'wwwroot', $wwwroot);
if (false == $institution) {
// This should never happen, because if we don't know the host we'll
......@@ -164,15 +162,23 @@ function fetch_user_image($username) {
} catch (Exception $e) {
// we don't care
continue;
}
}
}
if (count($candidates) != 1) {
return false;
}
$user = array_pop($candidates);
return array_pop($candidates);
}
function fetch_user_image($username) {
global $REMOTEWWWROOT;
if (!$user = find_remote_user($user, $REMOTEWWWROOT)) {
return false;
}
$ic = $user->profileicon;
if (!empty($ic)) {
$filename = get_config('dataroot') . 'artefact/internal/profileicons/' . ($user->profileicon % 256) . '/'.$user->profileicon;
......@@ -260,6 +266,79 @@ function user_authorise($token, $useragent) {
return $userdata;
}
function send_content_intent($username) {
global $REMOTEWWWROOT;
if (!$user = find_remote_user($username, $REMOTEWWWROOT)) {
// @todo return an error message we can understand
return false;
}
// @todo penny check for zip libraries here
// check whatever config values we have to check
// generate a token, insert it into the queue table
$usequeue = 0; // @todo change this (check whatever)
$queue = new StdClass;
$queue->token = generate_token();
$queue->host = $REMOTEWWWROOT;
$queue->usr = $user->id;
$queue->queue = $usequeue;
$queue->ready = 0;
$queue->expirytime = db_format_timestamp(time()+(60*60*24));
insert_record('import_queue', $queue);
return array(
'sendtype' => (($usequeue) ? 'queue' : 'immediate'),
'token' => $queue->token,
);
}
function send_content_ready($token, $username, $format, $filesmanifest, $fetchnow=false) {
global $REMOTEWWWROOT;
if (!$user = find_remote_user($username, $REMOTEWWWROOT)) {
throw new ImportException("Could not find user $username for $REMOTEWWWROOT");
}
// go verify the token
if (!$queue = get_record('import_queue', 'token', $token, 'host', $REMOTEWWWROOT)) {
throw new ImportException("Could not find queue record with token for username $username for $REMOTEWWWROOT");
}
if (strtotime($queue->expirytime) < time()) {
throw new ImportException("Queue record has expired");
}
// @todo penny verify format and filesmanifest
$queue->format = $format;
$queue->data = serialize(array('filesmanifest' => $filesmanifest));
update_record('import_queue', $queue);
$result = new StdClass;
// @todo penny change whatever we need here to match
// send_content_intent
if ($fetchnow && true) {
require_once('import.php');
// either immediately spawn a curl request to go fetch the file
$importer = Importer::create_importer($queue->id, $queue);
$importer->prepare();
$importer->process();
$result->status = true;
$result->type = 'complete';
} else {
// or set ready to 1 for the next cronjob to go fetch it.
$result->status = set_field('import_queue', 'ready', 1, 'id', $queue->id);
$result->type = 'queued';
}
log_debug($result);
return $result;
}
function xmlrpc_not_implemented() {
return true;
}
......@@ -453,7 +532,7 @@ function get_peer($wwwroot, $cache=true) {
if (!$peer->findByWwwroot($wwwroot)) {
// Bootstrap unknown hosts?
throw new MaharaException('We don\'t have a record for your webserver in our database', 6003);
throw new MaharaException("We don't have a record for your webserver ($wwwroot) in our database", 6003);
}
$peers[$wwwroot] = $peer;
return $peers[$wwwroot];
......
......@@ -656,13 +656,14 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
* Takes the name of a file outside the myfiles area.
* Returns a boolean indicating success or failure.
*/
public static function save_file($pathname, $data) {
public static function save_file($pathname, $data, User &$user=null) {
// This is only used when blog posts are saved: Files which
// have been uploaded to the post are moved to a permanent
// location in the files area using this function.
$dataroot = get_config('dataroot');
$pathname = $dataroot . $pathname;
if (!$size = filesize($pathname)) {
log_debug(1);
return false;
}
$f = self::new_file($pathname, $data);
......@@ -675,14 +676,26 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
$newname = $newdir . '/' . $id;
if (!rename($pathname, $newname)) {
$f->delete();
log_debug(2);
return false;
}
if (empty($user)) {
global $USER;
$user = $USER;
}
try {
$user->quota_add($size);
$user->commit();
return $id;
}
catch (QuotaExceededException $e) {
$f->delete();
log_debug(3);
return false;
}
global $USER;
$USER->quota_add($size);
$USER->commit();
return $id;
}
/**
* Processes a newly uploaded file, copies it to disk, and creates
* a new artefact object.
......
......@@ -120,6 +120,7 @@ class User {
}
$this->populate($user);
log_debug("after populate, quota is " . $this->get('quota') . ' and used is ' . $this->get('quotaused'));
$this->reset_institutions();
$this->reset_grouproles();
return $this;
......@@ -428,6 +429,7 @@ class User {
}
public function quota_allowed($bytes) {
log_debug("testing for " . $this->get('quotaused') . ' + ' . $bytes . ' being > than ' . $this->get('quota'));
if ($this->get('quotaused') + $bytes > $this->get('quota')) {
return false;
}
......
......@@ -60,6 +60,7 @@ require_once(get_config('libroot') .'institution.php');
$token = param_variable('token');
$remotewwwroot = param_variable('idp');
$wantsurl = param_variable('wantsurl', '/');
$remoteurl = param_boolean('remoteurl');
$institution = new Institution();
......@@ -106,6 +107,9 @@ if ($res == true) {
// Everything's ok - we have an authenticated User object
// confirm the MNET session
// redirect
if ($remoteurl) {
redirect($remotewwwroot . $wantsurl);
}
redirect(get_config('wwwroot') . $wantsurl);
// Redirect exits
}
......
......@@ -774,4 +774,9 @@ $string['done'] = 'Done';
$string['back'] = 'Back';
$string['alphabet'] = 'A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z';
// import related strings (maybe separated later)
$string['importedfrom'] = 'Imported from %s';
$string['incomingfolderdesc'] = 'Files imported from other networked hosts';
$string['remotehost'] = 'Remote host %s';
?>
......@@ -49,6 +49,7 @@ define('MAXRUNAGE', 300);
require(dirname(dirname(__FILE__)).'/init.php');
require_once(get_config('docroot') . 'artefact/lib.php');
require_once(get_config('docroot') . 'lib/import.php');
// This is here for debugging purposes, it allows us to fake the time to test
// cron behaviour
......
......@@ -685,6 +685,9 @@
<FIELD NAME="queue" TYPE="int" LENGTH="1" NOTNULL="true" default="1" />
<FIELD NAME="ready" TYPE="int" LENGTH="1" NOTNULL="true" default="0" />
<FIELD NAME="expirytime" TYPE="datetime" NOTNULL="true" />
<FIELD NAME="format" TYPE="char" LENGTH="50" NOTNULL="false" />
<FIELD NAME="data" TYPE="text" NOTNULL="false" />
<FIELD NAME="token" TYPE="char" LENGTH="40" NOTNULL="true"/>
</FIELDS>
<KEYS>
<KEY NAME="primary" TYPE="primary" FIELDS="id" />
......
......@@ -1223,18 +1223,31 @@ function xmldb_core_upgrade($oldversion=0) {
}
if ($oldversion < 2008081300) {
$table = new XMLDBTable('incoming_queue');
$table = new XMLDBTable('import_queue');
$table->addFieldInfo('id', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null, null, null);
$table->addFieldInfo('host', XMLDB_TYPE_CHAR, 255, null, XMLDB_NOTNULL);
$table->addFieldInfo('usr', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL);
$table->addFieldInfo('queue', XMLDB_TYPE_INTEGER, 1, null, XMLDB_NOTNULL, null, null, null, '1');
$table->addFieldInfo('ready', XMLDB_TYPE_INTEGER, 1, null, XMLDB_NOTNULL, null, null, null, '0');
$table->addFieldInfo('expirytime', XMLDB_TYPE_DATETIME, null, null, XMLDB_NOTNULL);
$table->addFieldInfo('format', XMLDB_TYPE_CHAR, 50, null, null);
$table->addFieldInfo('data', XMLDB_TYPE_TEXT, 'large', null, null);
$table->addFieldInfo('token', XMLDB_TYPE_CHAR, 40, null, XMLDB_NOTNULL);
$table->addKeyInfo('primary', XMLDB_KEY_PRIMARY, array('id'));
$table->addKeyInfo('usrfk', XMLDB_KEY_FOREIGN, array('usr'), 'usr', array('id'));
$table->addKeyInfo('hostfk', XMLDB_KEY_FOREIGN, array('host'), 'host', array('wwwroot'));
create_table($table);
// Install a cron job to process the queue
$cron = new StdClass;
$cron->callfunction = 'import_process_queue';
$cron->minute = '*/5';
$cron->hour = '*';
$cron->day = '*';
$cron->month = '*';
$cron->dayofweek = '*';
insert_record('cron', $cron);
}
return $status;
......
......@@ -780,5 +780,14 @@ class AccessDeniedException extends UserException {
}
}
/**
* something has happened during import.
* either: the user is there, in which case they get the bug screen,
* it's a spawned request during an xmlrpc server ping (content_ready) in which case XMLRPC will be defined
* or it's a queued fetch, in which case CRON will be defined.
* @todo maybe refactor at the point that we have something other than importing over mnet (eg userland)
*/
class ImportException extends SystemException { }
?>
<?php
/**
* Mahara: Electronic portfolio, weblog, resume builder and social networking
* Copyright (C) 2006-2008 Catalyst IT Ltd (http://www.catalyst.net.nz)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package mahara
* @subpackage core
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL
* @copyright (C) 2006-2008 Catalyst IT Ltd http://catalyst.net.nz
*/
defined('INTERNAL') || die();
function import_process_queue() {
if (!$ready = get_records_array('import_queue',
'ready', 1, null, null, null, null,
'*,' . db_format_tsfield('expirytime', 'ex'))) {
return true;
}
$now = time();
$processed = array();
foreach ($ready as $item) {
if ($item->ex < $now) {
log_debug('deleting expired import record', $item);
$processed[] = $item->id;
continue;
}
$importer = new MnetImporter($item->id, $item);
try {
$importer->process();
$processed[] = $item->id;
}
catch (Exception $e) {
$importer->cleanup();
}
}
if (empty($processed)) {
return true;
}
delete_records_select(
'incoming_queue',
'id IN ( ' . implode(',', db_array_to_ph($processed)) . ')',
$processed
);
}
function import_file($queue) {
$importer = new MnetImporter($queue->id, $queue);
$importer->process();
}
abstract class Importer {
private $id;
private $data;
private $host; // this might move
private $expirytime;
private $token;
private $usr;
private $usrobj;
private $importertransport;
public function __construct($id, $record=null) {
if (empty($record)) {
if (!$record = get_record('import_queue', 'id', $id)) {
throw new NotFoundException("Failed to find import queue record with id $id");
}
}
foreach ((array)$record as $field => $value) {
if ($field == 'data' && !is_array($value)) {
$value = unserialize($value);
}
$this->{$field} = $value;
}
$this->usrobj = new User();
$this->usrobj->find_by_id($this->usr);
if (!empty($this->host)) {
$this->importertransport = new MnetImporterTransport($this);
}
else {
$this->importertransport = new LocalImporterTransport($this);
}
// we could do more here later I guess
}
public function prepare() {
$this->importertransport->prepare_files();
}
/**
* processes the files and adds them to the user's artefact area
*/
public abstract function process();
public function get($field) {
if (!property_exists($this,$field)) {
throw new ParamOutOfRangeException("Field $field wasn't found in class " . get_class($this));
}
return $this->{$field};
}
public static function process_queue() {
if (!$ready = get_records_array('import_queue',
'ready', 1, null, null, null, null,
'*,' . db_format_tsfield('expirytime', 'ex'))) {
return true;
}
$now = time();
$processed = array();
foreach ($ready as $item) {
if ($item->ex < $now) {
log_debug('deleting expired import record', $item);
$processed[] = $item->id;
continue;
}
$importer = Importer::create_importer($item->id, $item);
try {
$importer->prepare();
$importer->process();
$processed[] = $item->id;
}
catch (Exception $e) {
$importer->cleanup();
}
}
if (empty($processed)) {
return true;
}
delete_records_select(
'incoming_queue',
'id IN ( ' . implode(',', db_array_to_ph($processed)) . ')',
$processed
);
}
public static function create_importer($id, $record=null) {
if (empty($record)) {
if (!$record = get_record('import_queue', 'id', $id)) {
throw new NotFoundException("Failed to find import queue record with id $id");
}
}
switch ($record->format) {
case 'file':
return new FilesImporter($id, $record);
default:
// @todo more laterz (like mahara native and/or leap)
throw new ImportException("unknown import format $record->format");
}
}
}
abstract class ImporterTransport {
/**
* this might be a path to a directory containing the files
* or an array containing some other info
* or the path to a file, depending on the format
*/
public abstract function files_info();
/**
* do whatever is necessary to retrieve the file(s)
*/
public abstract function prepare_files();
}
/**
* base case - just import files into the artefact area as files.
* don't interpret anything or try and create anything other than straight files.
*/
class FilesImporter extends Importer {
private $manifest;
private $files;
private $unzipdir;
public function __construct($id, $record=null) {
parent::__construct($id, $record);
$data = $this->get('data');
if (empty($data) ||
!is_array($data) ||
!array_key_exists('filesmanifest', $data) ||
!is_array($data['filesmanifest']) ||
count($data['filesmanifest']) == 0) {
throw new ImportException('Missing files manifest in import data');
}
$this->manifest = $data['filesmanifest'];
}
public function process() {
$this->extract_file();
$this->verify_file_contents();
$this->add_artefacts();
}
public function extract_file() {
$filesinfo = $this->get('importertransport')->files_info();
// this contains relativepath and zipfile name
$this->relativepath = $filesinfo['relativepath'];
$this->zipfile = $filesinfo['zipfile'];
$this->unzipdir = get_config('dataroot') . '/' . $this->relativepath . 'extract/';
if (!check_dir_exists($this->unzipdir)) {
throw new ImportException('Failed to create the temporary directories to work in');
}
// @todo penny maybe later replace this with a zip library
$command = "unzip " . escapeshellarg(get_config('dataroot') . '/' . $this->relativepath . '/' . $this->zipfile) . ' -d ' . escapeshellarg($this->unzipdir);
$output = array();
exec($command, $output, $returnvar);
if ($returnvar != 0) {
log_debug($output);
log_debug("return var $returnvar");
throw new ImportException('Failed to unzip the file recieved from the transport object');
}
}
public function verify_file_contents() {
$includedfiles = get_dir_contents($this->unzipdir);
$okfiles = array();
$badfiles = array();
// check what arrived in the directory
foreach ($includedfiles as $k => $f) {
// @todo penny later we might need this
if (is_dir($f)) {
$badfiles[] = $f;
unset($includedfiles[$k]);
continue;
}
$sha1 = sha1_file($this->unzipdir . $f);
if (array_key_exists($sha1, $this->manifest)) {
$tmp = new StdClass;
$tmp->sha1 = $sha1;
$tmp->wantsfilename = $this->manifest[$sha1]['filename'];
$tmp->actualfilename = $f;
$okfiles[] = $tmp;
unset($includedfiles[$k]);
continue;
}
$badfiles[] = $f;
unset($includedfiles[$k]);
}
$ok_c = count($okfiles);
$bad_c = count($badfiles);
$man_c = count($this->manifest);
if ($ok_c != $man_c) {
throw new ImportException('Files receieved did not exactly match what was in the manifest');
// @todo penny later - better reporting (missing files, too many files, etc)
}
$this->files = $okfiles;
}
public function add_artefacts() {
// we're just adding them as files into an 'incoming' directory in the user's file area.
safe_require('artefact', 'file');
try {
$dir = ArtefactTypeFolder::get_folder_id('incoming', get_string('incomingfolderdesc'), null, $this->get('usr'));
} catch (Exception $e) {
throw new ImportException($e->getMessage());
}
$savedfiles = array(); // to put files into so we can delete them should we encounter an exception
foreach ($this->files as $f) {
try {
$data = array(
'title' => $f->wantsfilename,