Commit 9a98b044 authored by Penny Leach's avatar Penny Leach
Browse files

Merge branch 'mahoodle-phase2'

parents 8929fc3f b080981f
......@@ -156,6 +156,7 @@ class Client {
throw new XmlrpcClientException('Remote site claims to have sent a public key, but they LIE');
}
}
throw new XmlrpcClientException('Unknown error occured: ' . $this->response['faultCode'] . ': ' . $this->response['faultString']);
}
// Clean up so object can be re-used.
......
......@@ -52,7 +52,16 @@ class Dispatcher {
'auth/mnet/auth.php/keepalive_server' => 'xmlrpc_not_implemented',
'auth/mnet/auth.php/kill_children' => 'xmlrpc_not_implemented',
'auth/mnet/auth.php/kill_child' => 'xmlrpc_not_implemented',
)
),
'portfolio_in' => array(
'portfolio/mahara/lib.php/send_content_intent' => 'send_content_intent',
'portfolio/mahara/lib.php/send_content_ready' => 'send_content_ready',
),
/* later...
'portfolio_out' => array(
),
*/
);
private $methodhelp = array(
......@@ -84,7 +93,14 @@ class Dispatcher {
'description' => 'username - The id of the user.'
)
)
)
),
'send_content_intent' => array(
array(
array('type' => 'string',
'description' => 'The username of the user on the remote system (previously sent in jump/land request)'
),
)
),
);
function __construct($payload, $payload_signed, $payload_encrypted) {
......
......@@ -121,10 +121,8 @@ function api_dummy_method($methodname, $argsarray, $functionname) {
return call_user_func_array($functionname, $argsarray);
}
function fetch_user_image($username) {
global $REMOTEWWWROOT;
$institution = get_field('host', 'institution', 'wwwroot', $REMOTEWWWROOT);
function find_remote_user($username, $wwwroot) {
$institution = get_field('host', 'institution', 'wwwroot', $wwwroot);
if (false == $institution) {
// This should never happen, because if we don't know the host we'll
......@@ -164,15 +162,23 @@ function fetch_user_image($username) {
} catch (Exception $e) {
// we don't care
continue;
}
}
}
if (count($candidates) != 1) {
return false;
}
$user = array_pop($candidates);
return array_pop($candidates);
}
function fetch_user_image($username) {
global $REMOTEWWWROOT;
if (!$user = find_remote_user($user, $REMOTEWWWROOT)) {
return false;
}
$ic = $user->profileicon;
if (!empty($ic)) {
$filename = get_config('dataroot') . 'artefact/internal/profileicons/' . ($user->profileicon % 256) . '/'.$user->profileicon;
......@@ -260,6 +266,86 @@ function user_authorise($token, $useragent) {
return $userdata;
}
function send_content_intent($username) {
global $REMOTEWWWROOT;
require_once('import.php');
if (!$user = find_remote_user($username, $REMOTEWWWROOT)) {
throw new ImportException("Could not find user $username for $REMOTEWWWROOT");
}
// @todo penny check for zip libraries here
// check whatever config values we have to check
// generate a token, insert it into the queue table
$usequeue = (int)!(Importer::import_immediately_allowed());
$queue = new StdClass;
$queue->token = generate_token();
$queue->host = $REMOTEWWWROOT;
$queue->usr = $user->id;
$queue->queue = $usequeue;
$queue->ready = 0;
$queue->expirytime = db_format_timestamp(time()+(60*60*24));
insert_record('import_queue', $queue);
return array(
'sendtype' => (($usequeue) ? 'queue' : 'immediate'),
'token' => $queue->token,
);
}
function send_content_ready($token, $username, $format, $importdata, $fetchnow=false) {
global $REMOTEWWWROOT;
require_once('import.php');
if (!$user = find_remote_user($username, $REMOTEWWWROOT)) {
throw new ImportException("Could not find user $username for $REMOTEWWWROOT");
}
// go verify the token
if (!$queue = get_record('import_queue', 'token', $token, 'host', $REMOTEWWWROOT)) {
throw new ImportException("Could not find queue record with given token for username $username for $REMOTEWWWROOT");
}
if (strtotime($queue->expirytime) < time()) {
throw new ImportException("Queue record has expired");
}
$queue->format = $format;
$class = null;
try {
$class = Importer::class_from_format($format);
} catch (Exception $e) {
throw new ImportException('Invalid format $format');
}
try {
call_static_method($class, 'validate_import_data', $importdata);
} catch (Exception $e) {
throw new ImportException('Invalid importdata: ' . $e->getMessage());
}
$queue->data = serialize($importdata);
update_record('import_queue', $queue);
$result = new StdClass;
if ($fetchnow && Importer::import_immediately_allowed()) {
// either immediately spawn a curl request to go fetch the file
$importer = Importer::create_importer($queue->id, $queue);
$importer->prepare();
$importer->process();
$result->status = true;
$result->type = 'complete';
} else {
// or set ready to 1 for the next cronjob to go fetch it.
$result->status = set_field('import_queue', 'ready', 1, 'id', $queue->id);
$result->type = 'queued';
}
return $result;
}
function xmlrpc_not_implemented() {
return true;
}
......@@ -453,7 +539,7 @@ function get_peer($wwwroot, $cache=true) {
if (!$peer->findByWwwroot($wwwroot)) {
// Bootstrap unknown hosts?
throw new MaharaException('We don\'t have a record for your webserver in our database', 6003);
throw new MaharaException("We don't have a record for your webserver ($wwwroot) in our database", 6003);
}
$peers[$wwwroot] = $peer;
return $peers[$wwwroot];
......
......@@ -700,13 +700,14 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
* Takes the name of a file outside the myfiles area.
* Returns a boolean indicating success or failure.
*/
public static function save_file($pathname, $data) {
public static function save_file($pathname, $data, User &$user=null) {
// This is only used when blog posts are saved: Files which
// have been uploaded to the post are moved to a permanent
// location in the files area using this function.
$dataroot = get_config('dataroot');
$pathname = $dataroot . $pathname;
if (!$size = filesize($pathname)) {
log_debug(1);
return false;
}
$f = self::new_file($pathname, $data);
......@@ -719,14 +720,26 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
$newname = $newdir . '/' . $id;
if (!rename($pathname, $newname)) {
$f->delete();
log_debug(2);
return false;
}
if (empty($user)) {
global $USER;
$user = $USER;
}
try {
$user->quota_add($size);
$user->commit();
return $id;
}
catch (QuotaExceededException $e) {
$f->delete();
log_debug(3);
return false;
}
global $USER;
$USER->quota_add($size);
$USER->commit();
return $id;
}
/**
* Processes a newly uploaded file, copies it to disk, and creates
* a new artefact object.
......
......@@ -60,6 +60,7 @@ require_once(get_config('libroot') .'institution.php');
$token = param_variable('token');
$remotewwwroot = param_variable('idp');
$wantsurl = param_variable('wantsurl', '/');
$remoteurl = param_boolean('remoteurl');
$institution = new Institution();
......@@ -106,6 +107,9 @@ if ($res == true) {
// Everything's ok - we have an authenticated User object
// confirm the MNET session
// redirect
if ($remoteurl) {
redirect($remotewwwroot . $wantsurl);
}
redirect(get_config('wwwroot') . $wantsurl);
// Redirect exits
}
......
......@@ -775,4 +775,9 @@ $string['done'] = 'Done';
$string['back'] = 'Back';
$string['alphabet'] = 'A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z';
// import related strings (maybe separated later)
$string['importedfrom'] = 'Imported from %s';
$string['incomingfolderdesc'] = 'Files imported from other networked hosts';
$string['remotehost'] = 'Remote host %s';
?>
......@@ -49,6 +49,7 @@ define('MAXRUNAGE', 300);
require(dirname(dirname(__FILE__)).'/init.php');
require_once(get_config('docroot') . 'artefact/lib.php');
require_once(get_config('docroot') . 'lib/import.php');
// This is here for debugging purposes, it allows us to fake the time to test
// cron behaviour
......
......@@ -692,5 +692,23 @@
<KEY NAME="viewfk" TYPE="foreign" FIELDS="view" REFTABLE="view" REFFIELDS="id" />
</KEYS>
</TABLE>
<TABLE NAME="import_queue">
<FIELDS>
<FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true" />
<FIELD NAME="host" TYPE="char" LENGTH="255" NOTNULL="true" />
<FIELD NAME="usr" TYPE="int" LENGTH="10" NOTNULL="true" />
<FIELD NAME="queue" TYPE="int" LENGTH="1" NOTNULL="true" default="1" />
<FIELD NAME="ready" TYPE="int" LENGTH="1" NOTNULL="true" default="0" />
<FIELD NAME="expirytime" TYPE="datetime" NOTNULL="true" />
<FIELD NAME="format" TYPE="char" LENGTH="50" NOTNULL="false" />
<FIELD NAME="data" TYPE="text" NOTNULL="false" />
<FIELD NAME="token" TYPE="char" LENGTH="40" NOTNULL="true"/>
</FIELDS>
<KEYS>
<KEY NAME="primary" TYPE="primary" FIELDS="id" />
<KEY NAME="usrfk" TYPE="foreign" FIELDS="usr" REFTABLE="usr" REFFIELDS="id" />
<KEY NAME="hostfk" TYPE="foreign" FIELDS="host" REFTABLE="host" REFFIELDS="wwwroot" />
</KEYS>
</TABLE>
</TABLES>
</XMLDB>
......@@ -1236,6 +1236,34 @@ function xmldb_core_upgrade($oldversion=0) {
);');
}
if ($oldversion < 2008090100) {
$table = new XMLDBTable('import_queue');
$table->addFieldInfo('id', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL, XMLDB_SEQUENCE, null, null, null);
$table->addFieldInfo('host', XMLDB_TYPE_CHAR, 255, null, XMLDB_NOTNULL);
$table->addFieldInfo('usr', XMLDB_TYPE_INTEGER, 10, null, XMLDB_NOTNULL);
$table->addFieldInfo('queue', XMLDB_TYPE_INTEGER, 1, null, XMLDB_NOTNULL, null, null, null, '1');
$table->addFieldInfo('ready', XMLDB_TYPE_INTEGER, 1, null, XMLDB_NOTNULL, null, null, null, '0');
$table->addFieldInfo('expirytime', XMLDB_TYPE_DATETIME, null, null, XMLDB_NOTNULL);
$table->addFieldInfo('format', XMLDB_TYPE_CHAR, 50, null, null);
$table->addFieldInfo('data', XMLDB_TYPE_TEXT, 'large', null, null);
$table->addFieldInfo('token', XMLDB_TYPE_CHAR, 40, null, XMLDB_NOTNULL);
$table->addKeyInfo('primary', XMLDB_KEY_PRIMARY, array('id'));
$table->addKeyInfo('usrfk', XMLDB_KEY_FOREIGN, array('usr'), 'usr', array('id'));
$table->addKeyInfo('hostfk', XMLDB_KEY_FOREIGN, array('host'), 'host', array('wwwroot'));
create_table($table);
// Install a cron job to process the queue
$cron = new StdClass;
$cron->callfunction = 'import_process_queue';
$cron->minute = '*/5';
$cron->hour = '*';
$cron->day = '*';
$cron->month = '*';
$cron->dayofweek = '*';
insert_record('cron', $cron);
}
return $status;
}
......
......@@ -780,5 +780,14 @@ class AccessDeniedException extends UserException {
}
}
/**
* something has happened during import.
* either: the user is there, in which case they get the bug screen,
* it's a spawned request during an xmlrpc server ping (content_ready) in which case XMLRPC will be defined
* or it's a queued fetch, in which case CRON will be defined.
* @todo maybe refactor at the point that we have something other than importing over mnet (eg userland)
*/
class ImportException extends SystemException { }
?>
<?php
/**
* Mahara: Electronic portfolio, weblog, resume builder and social networking
* Copyright (C) 2006-2008 Catalyst IT Ltd (http://www.catalyst.net.nz)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @package mahara
* @subpackage core
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL
* @copyright (C) 2006-2008 Catalyst IT Ltd http://catalyst.net.nz
*/
defined('INTERNAL') || die();
function import_process_queue() {
if (!$ready = get_records_array('import_queue',
'ready', 1, '',
'*,' . db_format_tsfield('expirytime', 'ex'))) {
return true;
}
$now = time();
$processed = array();
foreach ($ready as $item) {
if ($item->ex < $now) {
log_debug('deleting expired import record', $item);
$processed[] = $item->id;
continue;
}
$importer = Importer::create_importer($item->id, $item);
try {
$importer->prepare();
$importer->process();
$processed[] = $item->id;
}
catch (Exception $e) {
log_debug('an error occured on import: ' . $e->getMessage());
$importer->get('importertransport')->cleanup();
}
}
if (empty($processed)) {
return true;
}
delete_records_select(
'import_queue',
'id IN ( ' . implode(',', db_array_to_ph($processed)) . ')',
$processed
);
}
abstract class Importer {
private $id;
private $data;
private $host; // this might move
private $expirytime;
private $token;
private $usr;
private $usrobj;
private $importertransport;
public function __construct($id, $record=null) {
if (empty($record)) {
if (!$record = get_record('import_queue', 'id', $id)) {
throw new NotFoundException("Failed to find import queue record with id $id");
}
}
foreach ((array)$record as $field => $value) {
if ($field == 'data' && !is_array($value)) {
$value = unserialize($value);
}
$this->{$field} = $value;
}
$this->usrobj = new User();
$this->usrobj->find_by_id($this->usr);
if (!empty($this->host)) {
$this->importertransport = new MnetImporterTransport($this);
}
else {
$this->importertransport = new LocalImporterTransport($this);
}
// we could do more here later I guess
}
public function prepare() {
$this->importertransport->prepare_files();
}
/**
* processes the files and adds them to the user's artefact area
*/
public abstract function process();
public function get($field) {
if (!property_exists($this,$field)) {
throw new ParamOutOfRangeException("Field $field wasn't found in class " . get_class($this));
}
return $this->{$field};
}
public static function process_queue() {
if (!$ready = get_records_array('import_queue',
'ready', 1, null, null, null, null,
'*,' . db_format_tsfield('expirytime', 'ex'))) {
return true;
}
$now = time();
$processed = array();
foreach ($ready as $item) {
if ($item->ex < $now) {
log_debug('deleting expired import record', $item);
$processed[] = $item->id;
continue;
}
$importer = Importer::create_importer($item->id, $item);
try {
$importer->prepare();
$importer->process();
$processed[] = $item->id;
}
catch (Exception $e) {
$importer->cleanup();
}
}
if (empty($processed)) {
return true;
}
delete_records_select(
'incoming_queue',
'id IN ( ' . implode(',', db_array_to_ph($processed)) . ')',
$processed
);
}
public static function class_from_format($format) {
switch (trim($format)) {
case 'file':
case 'files':
return 'FilesImporter';
default:
// @todo more laterz (like mahara native and/or leap)
throw new ParamException("unknown import format $format");
}
}
public static function create_importer($id, $record=null) {
if (empty($record)) {
if (!$record = get_record('import_queue', 'id', $id)) {
throw new NotFoundException("Failed to find import queue record with id $id");
}
}
$class = self::class_from_format($record->format);
return new $class($id,$record);
}
public static abstract function validate_import_data($importdata);
public function import_immediately_allowed() {
// @todo change this (check whatever)
return true;
}
}
abstract class ImporterTransport {
/**
* this might be a path to a directory containing the files
* or an array containing some other info
* or the path to a file, depending on the format
*/
public abstract function files_info();
/**
* do whatever is necessary to retrieve the file(s)
*/
public abstract function prepare_files();
}
/**
* base case - just import files into the artefact area as files.
* don't interpret anything or try and create anything other than straight files.
*/
class FilesImporter extends Importer {
private $manifest;
private $files;
private $unzipdir;
private $zipfilesha1;
public function __construct($id, $record=null) {
parent::__construct($id, $record);
$data = $this->get('data');
self::validate_import_data($data);
$this->manifest = $data['filesmanifest'];
$this->zipfilesha1 = $data['zipfilesha1'];
}
public static function validate_import_data($importdata) {
if (empty($importdata) ||
!is_array($importdata) ||
!array_key_exists('filesmanifest', $importdata) ||
!is_array($importdata['filesmanifest']) ||
count($importdata['filesmanifest']) == 0) {
throw new ImportException('Missing files manifest in import data');
}
if (!array_key_exists('zipfilesha1', $importdata)) {
throw new ImportException('Missing zipfile sha1 in import data');
}
return true;
}
public function process() {
$this->extract_file();
$this->verify_file_contents();
$this->add_artefacts();
}
public function extract_file() {
$filesinfo = $this->get('importertransport')->files_info();
// this contains relativepath and zipfile name
$this->relativepath = $filesinfo['relativepath'];
$this->zipfile = $filesinfo['zipfile'];
if (sha1_file(get_config('dataroot') . '/' . $this->relativepath . '/' . $this->zipfile) != $this->zipfilesha1) {
throw new ImportException('sha1 of recieved zipfile didn\'t match expected sha1');
}
$this->unzipdir = get_config('dataroot') . '/' . $this->relativepath . 'extract/';
if (!check_dir_exists($this->unzipdir)) {
throw new ImportException('Failed to create the temporary directories to work in');
}
$command = sprintf('%s %s %s %s',
get_config('pathtounzip'),
escapeshellarg(get_config('dataroot') . '/' . $this->relativepath . '/' . $this->zipfile),
get_config('unzipdirarg'),
escapeshellarg($this->unzipdir)
);
$output = array();
exec($command, $output, $returnvar);
if ($returnvar != 0) {
throw new ImportException('Failed to unzip the file recieved from the transport object');
}
}
public function verify_file_contents() {
$includedfiles = get_dir_contents($this->unzipdir);
$okfiles = array();
$badfiles = array();
// check what arrived in the directory