Commit 78c87713 authored by Ilya Tregubov's avatar Ilya Tregubov Committed by Dmitrii Metelkin
Browse files

Bug 1685049: Remote file system modification

behatnotneeded

Enables Mahara to save files to an external file system
- object storage (such as AWS's S3) -
which can reduce the cost of storage

Change-Id: I76822612f2922ba0ef2a0b7a4efb9cd2b96979a6
parent e785dd1b
......@@ -11,11 +11,15 @@
<FIELD NAME="oldextension" TYPE="text" NOTNULL="false" />
<FIELD NAME="fileid" TYPE="int" LENGTH="10" NOTNULL="false" />
<FIELD NAME="filetype" TYPE="text" NOTNULL="false" />
<FIELD NAME="contenthash" TYPE="char" LENGTH="64" NOTNULL="false" />
</FIELDS>
<KEYS>
<KEY NAME="artefactpk" TYPE="primary" FIELDS="artefact" />
<KEY NAME="artefactfk" TYPE="foreign" FIELDS="artefact" REFTABLE="artefact" REFFIELDS="id" />
</KEYS>
<INDEXES>
<INDEX NAME="contenthashix" UNIQUE="false" FIELDS="contenthash"/>
</INDEXES>
</TABLE>
<TABLE NAME="artefact_file_image">
<FIELDS>
......
......@@ -483,5 +483,20 @@ function xmldb_artefact_file_upgrade($oldversion=0) {
}
}
if ($oldversion < 2017100901) {
require_once(get_config('docroot') . '/lib/file.php');
log_debug('Create a new "contenthash" field in "artefact_file_files" table');
$table = new XMLDBTable('artefact_file_files');
$field = new XMLDBField('contenthash');
$field->setAttributes(XMLDB_TYPE_CHAR, 64);
add_field($table, $field);
$index = new XMLDBIndex('contenthashix');
$index->setAttributes(XMLDB_INDEX_NOTUNIQUE, array('contenthash'));
add_index($table, $index);
}
return $status;
}
......@@ -155,6 +155,7 @@ function zip_process_contents(&$zip, $foldercontent, $path) {
$files = array_merge($files, zip_process_directory($zip, $content->id, $path . $content->title . '/'));
}
else {
$item->ensure_local();
$files[] = array($item->get_path(), $path . $item->download_title());
}
}
......
<?php
/**
* Interface external_file_system.
*
* @package mahara
* @subpackage artefact-internal
* @author Catalyst IT Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
* @copyright For copyright information on Mahara, please see the README file distributed with this software.
*
*/
defined('INTERNAL') || die();
/**
* External file systems should use these statuses for the files.
*
* @see external_file_system::get_file_location_status()
*/
define('FILE_LOCATION_ERROR', -1);
define('FILE_LOCATION_LOCAL', 0);
define('FILE_LOCATION_DUPLICATED', 1);
define('FILE_LOCATION_REMOTE', 2);
/**
* Describes an external file system class behavior.
* All external file system modules have to implement this interface.
*/
interface external_file_system {
/**
* Return an external file path.
*
* @param object $fileartefact This is the file object.
*
* @return string External path to the file
*/
public function get_path($fileartefact);
/**
* Check to see whether or not the external file is readable.
*
* @param object $fileartefact This is the file object.
*
* @return bool True if external file is readable, false otherwise
*/
public function is_file_readable($fileartefact);
/**
* Ensure that the file is local, copies the file from external
* to local if it is currently external.
*
* @param object $fileartefact This is the file object
*
* @return void
*/
public function ensure_local($fileartefact);
/**
* Return location status of a file. The file can be in 4 states:
*
* - FILE_LOCATION_ERROR - error status.
* - FILE_LOCATION_LOCAL - a file is stored locally only.
* - FILE_LOCATION_DUPLICATED - a file is stored locally and remotely.
* - FILE_LOCATION_REMOTE - a file is stored remotely only.
*
* @param object $fileartefact This is the file object.
*
* @return int status of file location
*/
public function get_file_location_status($fileartefact);
/**
* Copies a file from and external location to a local location.
*
* @param object $fileartefact This is the file object.
*
* @return int status of final file location
*/
public function copy_file_from_external_to_local($fileartefact);
/**
* Copies a file from a local location to an external location.
*
* @param object $fileartefact This is the file object.
*
* @return int status of final file location
*/
public function copy_file_from_local_to_external($fileartefact);
/**
* Delete the file.
*
* @param object $fileartefact This is the file object.
*
* @return int status of final file location
*/
public function delete_file($fileartefact);
/**
* Returns a file pointer resource of stream type.
*
* @param string $path File path.
* @param string $mode Parameter specifies the type of access you require to the stream. E.g. "r", "rb".
*
* @return resource
*/
public function get_file_handle($path, $mode);
}
......@@ -11,6 +11,8 @@
defined('INTERNAL') || die();
require_once('file.php');
class PluginArtefactFile extends PluginArtefact {
public static function get_artefact_types() {
......@@ -448,7 +450,8 @@ abstract class ArtefactTypeFileBase extends ArtefactType {
parent::__construct($id, $data);
if (empty($this->id)) {
$this->allowcomments = get_config_plugin('artefact', 'file', 'commentsallowed' . $this->artefacttype);
$allowcomments = get_config_plugin('artefact', 'file', 'commentsallowed' . $this->artefacttype);
$this->allowcomments = !is_null($allowcomments) ? $allowcomments : $this->allowcomments;
}
}
......@@ -477,7 +480,21 @@ abstract class ArtefactTypeFileBase extends ArtefactType {
return true;
}
/**
* Return an instance of external filesystem class or False if not configured.
*
* @return external_file_system
*/
final public static function get_external_filesystem_instance() {
global $CFG;
if (is_using_external_filesystem()) {
$classname = $CFG->externalfilesystem['class'];
return new $classname();
}
return false;
}
// Check if something exists in the db with a given title and parent,
// either in adminfiles or with a specific owner.
......@@ -1004,7 +1021,18 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
// file is a copy of another file artefact.
protected $fileid;
protected $filetype; // Mime type
// Mime type
protected $filetype;
// File content hash. It could be used by external file systems.
protected $contenthash;
/**
* Array with remote file system settings like 'includefilepath' and 'class'
*
* @var external_file_system
*/
protected $externalfilesystem;
public function __construct($id = 0, $data = null) {
parent::__construct($id, $data);
......@@ -1015,7 +1043,14 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
$this->{$name} = $value;
}
}
// If contenthash is not set for some reason, set it now.
if (empty($this->contenthash)) {
$this->save_content_hash();
}
}
$this->externalfilesystem = ArtefactTypeFileBase::get_external_filesystem_instance();
}
/**
......@@ -1051,12 +1086,17 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
'oldextension' => $this->get('oldextension'),
'fileid' => $this->get('fileid'),
'filetype' => $this->get('filetype'),
'contenthash' => $this->get('contenthash'),
);
if ($new) {
if (empty($data->fileid)) {
$data->fileid = $data->artefact;
}
if (empty($this->fileid)) {
$this->fileid = $data->fileid;
}
insert_record('artefact_file_files', $data);
}
else {
......@@ -1066,20 +1106,54 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
$this->dirty = false;
}
protected function save_content_hash() {
$this->contenthash = self::generate_content_hash($this->get_local_path());
if (!empty($this->contenthash)) {
$this->dirty = true;
$this->commit();
}
}
public static function generate_content_hash($path) {
if (!file_exists($path)) {
// We don't want to throw any exception here, because mahara doesn't catch them properly.
// Let's just return empty hash.
return '';
}
return hash_file('sha256', $path);
}
public static function get_file_directory($id) {
return "artefact/file/originals/" . ($id % 256);
}
public function get_path() {
public function get_path($data=array()) {
if (!empty($this->externalfilesystem)) {
return $this->externalfilesystem->get_path($this);
}
else {
return $this->get_local_path($data);
}
}
public function get_local_path($data = array()) {
return get_config('dataroot') . self::get_file_directory($this->fileid) . '/' . $this->fileid;
}
public function ensure_local() {
if (!empty($this->externalfilesystem)) {
$this->externalfilesystem->ensure_local($this);
}
}
/**
* Test file type and return a new Image or File.
*/
public static function new_file($path, $data) {
require_once('file.php');
$data->contenthash = self::generate_content_hash($path);
if (is_image_file($path)) {
// If it's detected as an image, overwrite the browser mime type
$imageinfo = getimagesize($path);
......@@ -1338,12 +1412,17 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
}
$file = $this->get_path();
if (is_file($file)) {
$size = filesize($file);
// Only delete the file on disk if no other artefacts point to it
if (count_records('artefact_file_files', 'fileid', $this->get('id')) == 1) {
unlink($file);
if (!empty($this->externalfilesystem)) {
// We trust an external filesystem to do it.
$this->externalfilesystem->delete_file($this);
}
else {
unlink($file);
}
}
global $USER;
// Deleting other users' files won't lower their quotas yet...
......@@ -1393,17 +1472,17 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
);
}
// Get all fileids so that we can delete the files on disk
$filetodeleteids = get_column_sql('
SELECT fileid
// Get all files so that we can delete the files on filesystem
$filerecords = get_records_sql_assoc('
SELECT aff1.*, a.artefacttype
FROM {artefact_file_files} aff1
JOIN {artefact} a ON aff1.artefact = a.id
WHERE artefact IN (' . $idstr . ')
GROUP BY fileid
HAVING COUNT(aff1.artefact) IN
(SELECT COUNT(aff2.artefact)
FROM {artefact_file_files} aff2
WHERE aff1.fileid = aff2.fileid)',
null
WHERE aff1.fileid = aff2.fileid)'
);
// The current rule is that file deletion should be logged in the artefact_log table
......@@ -1415,10 +1494,22 @@ class ArtefactTypeFile extends ArtefactTypeFileBase {
delete_records_select('artefact_file_files', 'artefact IN (' . $idstr . ')');
parent::bulk_delete($artefactids, $log);
foreach ($filetodeleteids as $filetodeleteid) {
$file = get_config('dataroot') . self::get_file_directory($filetodeleteid) . '/' . $filetodeleteid;
if (is_file($file)) {
unlink($file);
$externalfilesystem = ArtefactTypeFileBase::get_external_filesystem_instance();
if (!empty($filerecords)) {
foreach ($filerecords as $filerecord) {
if (!empty($externalfilesystem)) {
// We can't use artefact_instance_from_id() as we've removed all artefacts already
$classname = generate_artefact_class_name($filerecord->artefacttype);
$fileartefact = new $classname(0, $filerecord);
$externalfilesystem->delete_file($fileartefact);
}
else {
$file = get_config('dataroot') . self::get_file_directory($filerecord->fileid) . '/' . $filerecord->fileid;
if (is_file($file)) {
unlink($file);
}
}
}
}
......@@ -2288,10 +2379,8 @@ class ArtefactTypeImage extends ArtefactTypeFile {
return $url;
}
public function get_path($data=array()) {
require_once('file.php');
$result = get_dataroot_image_path('artefact/file/', $this->fileid, $data);
return $result;
public function get_local_path($data=array()) {
return get_dataroot_image_path('artefact/file/', $this->fileid, $data);
}
public function delete() {
......@@ -2390,10 +2479,8 @@ class ArtefactTypeProfileIcon extends ArtefactTypeImage {
return $url;
}
public function get_path($data=array()) {
require_once('file.php');
$result = get_dataroot_image_path('artefact/file/profileicons/', $this->fileid, $data);
return $result;
public function get_local_path($data=array()) {
return get_dataroot_image_path('artefact/file/profileicons/', $this->fileid, $data);
}
public function in_view_list() {
......@@ -2550,6 +2637,46 @@ class ArtefactTypeProfileIcon extends ArtefactTypeImage {
}
}
}
public static function save_uploaded_file($inputname, $data, $inputindex=null, $resized=false) {
global $USER;
require_once('uploadmanager.php');
$um = new upload_manager('file');
if ($error = $um->preprocess_file()) {
throw new UploadException($error);
}
$USER->quota_add($um->file['size']);
$imageinfo = getimagesize($inputname);
$data->parent = ArtefactTypeFolder::get_folder_id(get_string('imagesdir', 'artefact.file'), get_string('imagesdirdesc', 'artefact.file'), null, true, $USER->id);
$data->title = $data->title ? $data->title : $um->file['name'];
$data->title = ArtefactTypeFileBase::get_new_file_title($data->title, (int)$data->parent, $USER->id); // unique title
$data->owner = $USER->id;
$data->width = $imageinfo[0];
$data->height = $imageinfo[1];
$data->filetype = $imageinfo['mime'];
$data->description = get_string('uploadedprofileicon', 'artefact.file');
$data->contenthash = parent::generate_content_hash($inputname);
$data->size = $um->file['size'];
$data->note = $um->file['name'];
$data->oldextension = $um->original_filename_extension();
$artefact = new ArtefactTypeProfileIcon(0, $data);
$artefact->commit();
$id = $artefact->get('id');
// Move the file into the correct place.
$directory = get_config('dataroot') . 'artefact/file/profileicons/originals/' . ($id % 256) . '/';
check_dir_exists($directory);
move_uploaded_file($inputname, $directory . $id);
$USER->commit();
}
}
class ArtefactTypeArchive extends ArtefactTypeFile {
......
......@@ -240,46 +240,29 @@ function upload_validate(Pieform $form, $values) {
}
function upload_submit(Pieform $form, $values) {
global $USER, $filesize;
safe_require('artefact', 'file');
$data = new stdClass;
$data->title = $values['title'] ? $values['title'] : $values['file']['name'];
try {
$USER->quota_add($filesize);
ArtefactTypeProfileIcon::save_uploaded_file($values['file']['tmp_name'], $data);
}
catch (QuotaException $qe) {
catch (QuotaExceededException $e) {
$form->json_reply(PIEFORM_ERR, array(
'message' => get_string('profileiconuploadexceedsquota', 'artefact.file', get_config('wwwroot'))
));
}
// Entry in artefact table
$data = new stdClass;
$data->owner = $USER->id;
$data->parent = ArtefactTypeFolder::get_folder_id(get_string('imagesdir', 'artefact.file'), get_string('imagesdirdesc', 'artefact.file'), null, true, $USER->id);
$data->title = $values['title'] ? $values['title'] : $values['file']['name'];
$data->title = ArtefactTypeFileBase::get_new_file_title($data->title, (int)$data->parent, $USER->id); // unique title
$data->note = $values['file']['name'];
$data->size = $filesize;
$imageinfo = getimagesize($values['file']['tmp_name']);
$data->width = $imageinfo[0];
$data->height = $imageinfo[1];
$data->filetype = $imageinfo['mime'];
$data->description = get_string('uploadedprofileicon', 'artefact.file');
$artefact = new ArtefactTypeProfileIcon(0, $data);
if (preg_match("/\.([^\.]+)$/", $values['file']['name'], $saved)) {
$artefact->set('oldextension', $saved[1]);
catch (UploadException $e) {
$form->json_reply(PIEFORM_ERR, array(
'message' => get_string('uploadoffilefailed', 'artefact.file', $data->title) . ': ' . $e->getMessage()
));
}
catch (Exception $e) {
$form->json_reply(PIEFORM_ERR, array(
'message' => get_string('uploadoffilefailed', 'artefact.file', $data->title) . ': ' . $e->getMessage()
));
}
$artefact->commit();
$id = $artefact->get('id');
// Move the file into the correct place.
$directory = get_config('dataroot') . 'artefact/file/profileicons/originals/' . ($id % 256) . '/';
check_dir_exists($directory);
move_uploaded_file($values['file']['tmp_name'], $directory . $id);
$USER->commit();
$form->json_reply(PIEFORM_OK, get_string('profileiconaddedtoimagesfolder', 'artefact.file', get_string('imagesdir', 'artefact.file')));
}
......
......@@ -12,6 +12,6 @@
defined('INTERNAL') || die();
$config = new StdClass;
$config->version = 2017100900;
$config->release = '1.2.8';
$config->version = 2017100901;
$config->release = '1.2.9';
......@@ -465,6 +465,28 @@ if (!defined('INSTALLER')) {
if (get_config('disableexternalresources')) {
$CFG->wwwhost = parse_url($CFG->wwwroot, PHP_URL_HOST);
}
// Ensure that externalfilesystem is configured correctly.
if (get_config('externalfilesystem')) {
if (empty($CFG->externalfilesystem['includefilepath']) || empty($CFG->externalfilesystem['class'])) {
throw new ConfigSanityException('externalfilesystem configuration detected but the settings are invalid');
}
if (!file_exists($CFG->docroot . '/' . $CFG->externalfilesystem['includefilepath'])) {
throw new ConfigSanityException('externalfilesystem is configured, but file ' . $CFG->externalfilesystem['includefilepath'] . ' does not exist');
}
require_once($CFG->docroot . '/' . $CFG->externalfilesystem['includefilepath']);
if (!class_exists($CFG->externalfilesystem['class'])) {
throw new ConfigSanityException('externalfilesystem is configured, but class ' . $CFG->externalfilesystem['class'] . ' does not exist');
}
if (!is_subclass_of($CFG->externalfilesystem['class'], 'external_file_system')) {
throw new ConfigSanityException('externalfilesystem is configured, but ' . $CFG->externalfilesystem['class'] . ' class does not implement external_file_system interface');
}
}
/*
* Initializes our performance info early.
*
......
......@@ -740,3 +740,14 @@ $cfg->openbadgedisplayer_source = '{"backpack":"https://backpack.openbadges.org/
"sp" : ["' . $cfg->dataroot . '/customattributemap/customname2oid.php"]
}';
*/
/**
* @global array $cfg->externalfilesystem
* A configuration data for an external file system
*/
/*
$cfg->externalfilesystem = '{
"includefilepath" : "module/objectfs/classes/s3_file_system.php",
"class" : "s3_file_system",
}';
*/
......@@ -36,14 +36,17 @@ define('BYTESERVING_BOUNDARY', 'm1i2k3e40516'); //unique string constant
* there are none.
*/
function serve_file($path, $filename, $mimetype, $options=array()) {
$dataroot = realpath(get_config('dataroot'));
$path = realpath($path);
$options = array_merge(array(
'lifetime' => 86400
), $options);
if (!get_config('insecuredataroot') && substr($path, 0, strlen($dataroot)) != $dataroot) {
throw new AccessDeniedException();
if (!is_using_external_filesystem()) {
$dataroot = realpath(get_config('dataroot'));
$localpath = realpath($path);
if (!get_config('insecuredataroot') && substr($localpath, 0, strlen($dataroot)) != $dataroot) {
throw new AccessDeniedException();
}
}
if (!file_exists($path)) {
......@@ -172,7 +175,7 @@ function readfile_chunked($filename, $retbytes=true) {
$chunksize = 1 * (1024 * 1024); // 1MB chunks - must be less than 2MB!
$buffer = '';
$cnt =0;
$handle = fopen($filename, 'rb');
$handle = get_file_handle($filename, 'rb');
if ($handle === false) {
return false;
}
......@@ -199,7 +202,7 @@ function readfile_chunked($filename, $retbytes=true) {
*/
function byteserving_send_file($filename, $mimetype, $ranges) {
$chunksize = 1 * (1024 * 1024); // 1MB chunks - must be less than 2MB!
$handle = fopen($filename, 'rb');
$handle = get_file_handle($filename, 'rb');
if ($handle === false) {
die;
}
......@@ -252,6 +255,29 @@ function byteserving_send_file($filename, $mimetype, $ranges) {
}
}
/**
* Return file handle.