Commit 97ac2f60 authored by Robert Lyon's avatar Robert Lyon Committed by Gerrit Code Review

Merge "Bug 1685049: Remote file system modification"

parents 07f2b0f5 78c87713
......@@ -11,11 +11,15 @@
<FIELD NAME="oldextension" TYPE="text" NOTNULL="false" />
<FIELD NAME="fileid" TYPE="int" LENGTH="10" NOTNULL="false" />
<FIELD NAME="filetype" TYPE="text" NOTNULL="false" />
<FIELD NAME="contenthash" TYPE="char" LENGTH="64" NOTNULL="false" />
</FIELDS>
<KEYS>
<KEY NAME="artefactpk" TYPE="primary" FIELDS="artefact" />
<KEY NAME="artefactfk" TYPE="foreign" FIELDS="artefact" REFTABLE="artefact" REFFIELDS="id" />
</KEYS>
<INDEXES>
<INDEX NAME="contenthashix" UNIQUE="false" FIELDS="contenthash"/>
</INDEXES>
</TABLE>
<TABLE NAME="artefact_file_image">
<FIELDS>
......
......@@ -483,5 +483,20 @@ function xmldb_artefact_file_upgrade($oldversion=0) {
}
}
if ($oldversion < 2017100901) {
require_once(get_config('docroot') . '/lib/file.php');
log_debug('Create a new "contenthash" field in "artefact_file_files" table');
$table = new XMLDBTable('artefact_file_files');
$field = new XMLDBField('contenthash');
$field->setAttributes(XMLDB_TYPE_CHAR, 64);
add_field($table, $field);
$index = new XMLDBIndex('contenthashix');
$index->setAttributes(XMLDB_INDEX_NOTUNIQUE, array('contenthash'));
add_index($table, $index);
}
return $status;
}
......@@ -155,6 +155,7 @@ function zip_process_contents(&$zip, $foldercontent, $path) {
$files = array_merge($files, zip_process_directory($zip, $content->id, $path . $content->title . '/'));
}
else {
$item->ensure_local();
$files[] = array($item->get_path(), $path . $item->download_title());
}
}
......
<?php
/**
* Interface external_file_system.
*
* @package mahara
* @subpackage artefact-internal
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
defined('INTERNAL') || die();
/**
* External file systems should use these statuses for the files.
*
* @see external_file_system::get_file_location_status()
*/
define('FILE_LOCATION_ERROR', -1);
define('FILE_LOCATION_LOCAL', 0);
define('FILE_LOCATION_DUPLICATED', 1);
define('FILE_LOCATION_REMOTE', 2);
/**
* Describes an external file system class behavior.
* All external file system modules have to implement this interface.
*/
interface external_file_system {
/**
* Return an external file path.
*
* @param object $fileartefact This is the file object.
*
* @return string External path to the file
*/
public function get_path($fileartefact);
/**
* Check to see whether or not the external file is readable.
*
* @param object $fileartefact This is the file object.
*
* @return bool True if external file is readable, false otherwise
*/
public function is_file_readable($fileartefact);
/**
* Ensure that the file is local, copies the file from external
* to local if it is currently external.
*
* @param object $fileartefact This is the file object
*
* @return void
*/
public function ensure_local($fileartefact);
/**
* Return location status of a file. The file can be in 4 states:
*
* - FILE_LOCATION_ERROR - error status.
* - FILE_LOCATION_LOCAL - a file is stored locally only.
* - FILE_LOCATION_DUPLICATED - a file is stored locally and remotely.
* - FILE_LOCATION_REMOTE - a file is stored remotely only.
*
* @param object $fileartefact This is the file object.
*
* @return int status of file location
*/
public function get_file_location_status($fileartefact);
/**
* Copies a file from and external location to a local location.
*
* @param object $fileartefact This is the file object.
*
* @return int status of final file location
*/
public function copy_file_from_external_to_local($fileartefact);
/**
* Copies a file from a local location to an external location.
*
* @param object $fileartefact This is the file object.
*
* @return int status of final file location
*/
public function copy_file_from_local_to_external($fileartefact);
/**
* Delete the file.
*
* @param object $fileartefact This is the file object.
*
* @return int status of final file location
*/
public function delete_file($fileartefact);
/**
* Returns a file pointer resource of stream type.
*
* @param string $path File path.
* @param string $mode Parameter specifies the type of access you require to the stream. E.g. "r", "rb".
*
* @return resource
*/
public function get_file_handle($path, $mode);
}
This diff is collapsed.
......@@ -240,46 +240,29 @@ function upload_validate(Pieform $form, $values) {
}
function upload_submit(Pieform $form, $values) {
global $USER, $filesize;
safe_require('artefact', 'file');
$data = new stdClass;
$data->title = $values['title'] ? $values['title'] : $values['file']['name'];
try {
$USER->quota_add($filesize);
ArtefactTypeProfileIcon::save_uploaded_file($values['file']['tmp_name'], $data);
}
catch (QuotaException $qe) {
catch (QuotaExceededException $e) {
$form->json_reply(PIEFORM_ERR, array(
'message' => get_string('profileiconuploadexceedsquota', 'artefact.file', get_config('wwwroot'))
));
}
// Entry in artefact table
$data = new stdClass;
$data->owner = $USER->id;
$data->parent = ArtefactTypeFolder::get_folder_id(get_string('imagesdir', 'artefact.file'), get_string('imagesdirdesc', 'artefact.file'), null, true, $USER->id);
$data->title = $values['title'] ? $values['title'] : $values['file']['name'];
$data->title = ArtefactTypeFileBase::get_new_file_title($data->title, (int)$data->parent, $USER->id); // unique title
$data->note = $values['file']['name'];
$data->size = $filesize;
$imageinfo = getimagesize($values['file']['tmp_name']);
$data->width = $imageinfo[0];
$data->height = $imageinfo[1];
$data->filetype = $imageinfo['mime'];
$data->description = get_string('uploadedprofileicon', 'artefact.file');
$artefact = new ArtefactTypeProfileIcon(0, $data);
if (preg_match("/\.([^\.]+)$/", $values['file']['name'], $saved)) {
$artefact->set('oldextension', $saved[1]);
catch (UploadException $e) {
$form->json_reply(PIEFORM_ERR, array(
'message' => get_string('uploadoffilefailed', 'artefact.file', $data->title) . ': ' . $e->getMessage()
));
}
catch (Exception $e) {
$form->json_reply(PIEFORM_ERR, array(
'message' => get_string('uploadoffilefailed', 'artefact.file', $data->title) . ': ' . $e->getMessage()
));
}
$artefact->commit();
$id = $artefact->get('id');
// Move the file into the correct place.
$directory = get_config('dataroot') . 'artefact/file/profileicons/originals/' . ($id % 256) . '/';
check_dir_exists($directory);
move_uploaded_file($values['file']['tmp_name'], $directory . $id);
$USER->commit();
$form->json_reply(PIEFORM_OK, get_string('profileiconaddedtoimagesfolder', 'artefact.file', get_string('imagesdir', 'artefact.file')));
}
......
......@@ -12,6 +12,6 @@
defined('INTERNAL') || die();
$config = new StdClass;
$config->version = 2017100900;
$config->release = '1.2.8';
$config->version = 2017100901;
$config->release = '1.2.9';
......@@ -465,6 +465,28 @@ if (!defined('INSTALLER')) {
if (get_config('disableexternalresources')) {
$CFG->wwwhost = parse_url($CFG->wwwroot, PHP_URL_HOST);
}
// Ensure that externalfilesystem is configured correctly.
if (get_config('externalfilesystem')) {
if (empty($CFG->externalfilesystem['includefilepath']) || empty($CFG->externalfilesystem['class'])) {
throw new ConfigSanityException('externalfilesystem configuration detected but the settings are invalid');
}
if (!file_exists($CFG->docroot . '/' . $CFG->externalfilesystem['includefilepath'])) {
throw new ConfigSanityException('externalfilesystem is configured, but file ' . $CFG->externalfilesystem['includefilepath'] . ' does not exist');
}
require_once($CFG->docroot . '/' . $CFG->externalfilesystem['includefilepath']);
if (!class_exists($CFG->externalfilesystem['class'])) {
throw new ConfigSanityException('externalfilesystem is configured, but class ' . $CFG->externalfilesystem['class'] . ' does not exist');
}
if (!is_subclass_of($CFG->externalfilesystem['class'], 'external_file_system')) {
throw new ConfigSanityException('externalfilesystem is configured, but ' . $CFG->externalfilesystem['class'] . ' class does not implement external_file_system interface');
}
}
/*
* Initializes our performance info early.
*
......
......@@ -749,3 +749,14 @@ $cfg->sessionhandler = 'file';
"sp" : ["' . $cfg->dataroot . '/customattributemap/customname2oid.php"]
}';
*/
/**
* @global array $cfg->externalfilesystem
* A configuration data for an external file system
*/
/*
$cfg->externalfilesystem = '{
"includefilepath" : "module/objectfs/classes/s3_file_system.php",
"class" : "s3_file_system",
}';
*/
......@@ -36,14 +36,17 @@ define('BYTESERVING_BOUNDARY', 'm1i2k3e40516'); //unique string constant
* there are none.
*/
function serve_file($path, $filename, $mimetype, $options=array()) {
$dataroot = realpath(get_config('dataroot'));
$path = realpath($path);
$options = array_merge(array(
'lifetime' => 86400
), $options);
if (!get_config('insecuredataroot') && substr($path, 0, strlen($dataroot)) != $dataroot) {
throw new AccessDeniedException();
if (!is_using_external_filesystem()) {
$dataroot = realpath(get_config('dataroot'));
$localpath = realpath($path);
if (!get_config('insecuredataroot') && substr($localpath, 0, strlen($dataroot)) != $dataroot) {
throw new AccessDeniedException();
}
}
if (!file_exists($path)) {
......@@ -172,7 +175,7 @@ function readfile_chunked($filename, $retbytes=true) {
$chunksize = 1 * (1024 * 1024); // 1MB chunks - must be less than 2MB!
$buffer = '';
$cnt =0;
$handle = fopen($filename, 'rb');
$handle = get_file_handle($filename, 'rb');
if ($handle === false) {
return false;
}
......@@ -199,7 +202,7 @@ function readfile_chunked($filename, $retbytes=true) {
*/
function byteserving_send_file($filename, $mimetype, $ranges) {
$chunksize = 1 * (1024 * 1024); // 1MB chunks - must be less than 2MB!
$handle = fopen($filename, 'rb');
$handle = get_file_handle($filename, 'rb');
if ($handle === false) {
die;
}
......@@ -252,6 +255,29 @@ function byteserving_send_file($filename, $mimetype, $ranges) {
}
}
/**
* Return file handle.
*
* @param string $path A file path.
* @param string $mode Parameter specifies the type of access you require to the stream. E.g. "r", "rb".
*
* @return resource
*/
function get_file_handle($path, $mode) {
if (is_using_external_filesystem()) {
$externalfilesystem = ArtefactTypeFileBase::get_external_filesystem_instance();
$handle = $externalfilesystem->get_file_handle($path, $mode);
if (!is_resource($handle) || get_resource_type($handle) != 'stream') {
$handle = false;
}
}
else {
$handle = fopen($path, $mode);
}
return $handle;
}
/**
* Given a file path, guesses the mime type of the file using the
......@@ -503,6 +529,14 @@ function get_dataroot_image_path($path, $id, $size=null) {
// Work out the location of the original image
$originalimage = $imagepath . '/originals/' . ($id % 256) . "/$id";
// We have to have an original image locally to be able to build preview,
// because image functions like getimagesize do not work with stream wrappers
// So if the original image is not readable, try to download it from an external system (if enabled).
if (!is_readable($originalimage) && is_using_external_filesystem()) {
$image = artefact_instance_from_id($id);
$image->ensure_local();
}
// If the original has been deleted, then don't show any image, even a cached one.
// delete_image only deletes the original, not any cached ones, so we have
// to make sure the original is still around
......@@ -959,3 +993,12 @@ function xml_filter_regex() {
.']/u';
return $regex;
}
/**
* Check if we are using external file system.
*
* @return bool
*/
function is_using_external_filesystem() {
return get_config('externalfilesystem', false);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment