Bug 1747297: Adding Redis session storage option

Taking work developed by Matt C and adjusting it to be more of a
session handler option.

behatnotneeded

Change-Id: I443d9e67d2322c995f9277ce1ee835092dc6c218
Signed-off-by: Robert Lyon <robertl@catalyst.net.nz>

Makefile

@@ -57,13 +57,13 @@ cleanssphp:
 	@echo "Cleaning out SimpleSAMLphp..."
 	rm -rf htdocs/auth/saml/extlib/simplesamlphp
 
-ssphp:
+ssphp: initcomposer
 ifdef simplesamlphp
 	@echo "SimpleSAMLphp already exists - doing nothing"
 else
 	@echo "Pulling SimpleSAMLphp from download ..."
-	@curl -sSL https://github.com/simplesamlphp/simplesamlphp/releases/download/v1.15.0/simplesamlphp-1.15.0.tar.gz | tar  --transform 's/simplesamlphp-[0-9]+\.[0-9]+\.[0-9]+/simplesamlphp/x1' -C htdocs/auth/saml/extlib -xzf - # SimpleSAMLPHP release tarball already has all composer dependencies.
-#	@php external/composer.phar --working-dir=htdocs/auth/saml/extlib/simplesamlphp update --no-dev
+	@curl -sSL https://github.com/simplesamlphp/simplesamlphp/releases/download/v1.15.1/simplesamlphp-1.15.1.tar.gz | tar  --transform 's/simplesamlphp-[0-9]+\.[0-9]+\.[0-9]+/simplesamlphp/x1' -C htdocs/auth/saml/extlib -xzf - # SimpleSAMLPHP release tarball already has all composer dependencies.
+	@php external/composer.phar --working-dir=htdocs/auth/saml/extlib/simplesamlphp require predis/predis
 	@echo "Copying www/resources/* files to sp/resources/ ..."
 	@cp -R htdocs/auth/saml/extlib/simplesamlphp/www/resources/ htdocs/auth/saml/sp/
 	@echo "Deleting unneeded files ..."

htdocs/auth/saml/config/config.php

@@ -31,13 +31,18 @@ foreach ($metadata_files as $file) {
 
 // Fix up session handling config - to match Mahara
 $memcache_config = array();
+$redis_config = array('host' => '', 'port' => 6379, 'prefix' => '');
 if (empty(get_config('ssphpsessionhandler'))) {
     if (PluginAuthSaml::is_memcache_configured()) {
         $sessionhandler = 'memcache';
         $memcache_config = PluginAuthSaml::get_memcache_servers();
     }
+    else if (PluginAuthSaml::is_redis_configured()) {
+        $sessionhandler = 'redis';
+        $redis_config = PluginAuthSaml::get_redis_config();
+    }
     else {
-        throw new AuthInstanceException(get_string('errornomemcache', 'auth.saml'));
+        throw new AuthInstanceException(get_string('errornovalidsessionhandler', 'auth.saml'));
     }
 }
 else {
@@ -356,20 +361,6 @@ $config = array (
 
     'metadata.sources' => $metadata_sources,
 
-
-    /*
-     * This configuration option allows you to select which session handler
-     * SimpleSAMLPHP should use to store the session information. Currently
-     * we have two session handlers:
-     * - 'phpsession': The default PHP session handler.
-     * - 'memcache': Stores the session information in one or more
-     *   memcache servers by using the MemcacheStore class.
-     *
-     * The default session handler is 'phpsession'.
-     */
-    'session.handler'       => $sessionhandler,
-
-
     /*
      * Configuration for the MemcacheStore class. This allows you to store
      * multiple redudant copies of sessions on different memcache servers.
@@ -443,14 +434,16 @@ $config = array (
      */
     'memcache_store.expires' =>  60,
 
+    /*
+     * The hostname and port of the Redis datastore instance.
+     */
+    'store.redis.host' => $redis_config['host'],
+    'store.redis.port' => $redis_config['port'],
 
-    'redis_store.servers' => array(
-        array(
-            array('hostname' => 'localhost'),
-        ),
-    ),
-    'redis_store.expires' =>  36 * (60*60), // 36 hours.
-
+    /*
+     * The prefix we should use on our Redis datastore.
+     */
+    'store.redis.prefix' => $redis_config['prefix'],
 
     /*
      * Should signing of generated metadata be enabled by default.
@@ -480,6 +473,53 @@ $config = array (
     'metadata.sign.privatekey_pass' => NULL,
     'metadata.sign.certificate' => NULL,
 
+     /****************************
+     | DATA STORE CONFIGURATION |
+     ****************************/
+
+    /*
+     * Configure the data store for SimpleSAMLphp.
+     *
+     * - 'phpsession': Limited datastore, which uses the PHP session.
+     * - 'memcache': Key-value datastore, based on memcache.
+     * - 'sql': SQL datastore, using PDO.
+     * - 'redis': Key-value datastore, based on redis.
+     *
+     * The default datastore is 'phpsession'.
+     *
+     * (This option replaces the old 'session.handler'-option.)
+     */
+    'store.type'=> $sessionhandler,
+
+    /*
+     * The DSN the sql datastore should connect to.
+     *
+     * See http://www.php.net/manual/en/pdo.drivers.php for the various
+     * syntaxes.
+     */
+    'store.sql.dsn'       => 'sqlite:/path/to/sqlitedatabase.sq3',
+
+    /*
+     * The username and password to use when connecting to the database.
+     */
+    'store.sql.username' => null,
+    'store.sql.password' => null,
+
+    /*
+     * The prefix we should use on our tables.
+     */
+    'store.sql.prefix' => 'SimpleSAMLphp',
+
+    /*
+     * The hostname and port of the Redis datastore instance.
+     */
+    'store.redis.host' => $redis_config['host'],
+    'store.redis.port' => $redis_config['port'],
+
+    /*
+     * The prefix we should use on our Redis datastore.
+     */
+    'store.redis.prefix' => $redis_config['prefix'],
 );
 
 // if we set custom mappings files paths in config.php

htdocs/auth/saml/db/upgrade.php

@@ -34,5 +34,10 @@ function xmldb_auth_saml_upgrade($oldversion=0) {
         set_config_plugin('auth', 'saml', 'version', '1.15.0');
     }
 
+    if ($oldversion < 2018021600) {
+        // Set library version to download
+        set_config_plugin('auth', 'saml', 'version', '1.15.1');
+    }
+
     return $status;
 }

htdocs/auth/saml/index.php

@@ -40,10 +40,6 @@ if (get_field('auth_installed', 'active', 'name', 'saml') != 1) {
     redirect();
 }
 
-if (!extension_loaded('mcrypt')) {
-    throw new AuthInstanceException(get_string_php_version('errornomcrypt','auth.saml'));
-}
-
 $sp = 'default-sp';
 
 PluginAuthSaml::init_simplesamlphp();
@@ -51,6 +47,9 @@ PluginAuthSaml::init_simplesamlphp();
 // Check the SimpleSAMLphp config is compatible
 $saml_config = SimpleSAML_Configuration::getInstance();
 $session_handler = $saml_config->getString('session.handler', false);
+if ($session_handler == 'memcache' && !extension_loaded('mcrypt')) {
+    throw new AuthInstanceException(get_string_php_version('errornomcrypt','auth.saml'));
+}
 $store_type = $saml_config->getString('store.type', false);
 if ($store_type == 'phpsession' || $session_handler == 'phpsession' || (empty($store_type) && empty($session_handler))) {
     throw new AuthInstanceException(get_string('errorbadssphp', 'auth.saml'));

htdocs/auth/saml/lang/en.utf8/auth.saml.php

@@ -35,6 +35,7 @@ $string['errorbadlib'] = 'The SimpleSAMLPHP library\'s "autoloader" file was not
 $string['errorupdatelib'] = 'Your current SimpleSAMLPHP library version is out of date. You need to run "make cleanssphp && make ssphp".';
 $string['errornomcrypt'] = 'The PHP library "mcrypt" must be installed for auth/saml. Make sure you install and activate mcrypt, e.g.:<br>sudo apt-get install php5-mcrypt<br>sudo php5enmod mcrypt<br>Then restart your web server.';
 $string['errornomcrypt7php'] = 'The PHP library "mcrypt" must be installed for auth/saml. Make sure you install and activate mcrypt, e.g.:<br>sudo apt-get install php7.0-mcrypt<br>sudo phpenmod mcrypt<br>Then restart your web server.';
+$string['errornovalidsessionhandler'] = 'The SimpleSAMLphp session handler is misconfigured or the server is currently unavailable.';
 $string['errornomemcache'] = 'Memcache is misconfigured for auth/saml or a Memcache server is currently unavailable.';
 $string['errornomemcache7php'] = 'Memcache is misconfigured for auth/saml or a Memcache server is currently unavailable.';
 $string['errorbadconfig'] = 'The SimpleSAMLPHP config directory %s is incorrect.';

htdocs/auth/saml/lib.php

@@ -367,7 +367,7 @@ class PluginAuthSaml extends PluginAuth {
 
     public static function install_auth_default() {
         // Set library version to download
-        set_config_plugin('auth', 'saml', 'version', '1.15.0');
+        set_config_plugin('auth', 'saml', 'version', '1.15.1');
     }
 
     private static function create_certificates($numberofdays = 3650) {
@@ -574,7 +574,7 @@ class PluginAuthSaml extends PluginAuth {
         // check extensions are loaded
         $libchecks = '';
         // Make sure mcrypt exists
-        if (!extension_loaded('mcrypt')) {
+        if (get_config('memcacheservers') && !extension_loaded('mcrypt')) {
             $libchecks .= '<li>' . get_string_php_version('errornomcrypt', 'auth.saml') . '</li>';
         }
         // Make sure the simplesamlphp files have been installed via 'make ssphp'
@@ -592,9 +592,9 @@ class PluginAuthSaml extends PluginAuth {
                 $libchecks .= '<li>' . get_string('errorupdatelib', 'auth.saml') . '</li>';
             }
         }
-        // Make sure we can use 'memcache' with simplesamlphp as 'phpsession' doesn't work correctly in many situations
-        if (!self::is_memcache_configured()) {
-            $libchecks .= '<li>' . get_string_php_version('errornomemcache', 'auth.saml') . '</li>';
+        // Make sure we can use a valid session handler with simplesamlphp as 'phpsession' doesn't work correctly in many situations
+        if (!self::is_usable()) {
+            $libchecks .= '<li>' . get_string_php_version('errornovalidsessionhandler', 'auth.saml') . '</li>';
         }
         if (!empty($libchecks)) {
             $libcheckstr = '<div class="alert alert-danger"><ul class="unstyled">' . $libchecks . '</ul></div>';
@@ -637,11 +637,23 @@ class PluginAuthSaml extends PluginAuth {
             return false;
         }
 
+        if (get_config('ssphpsessionhandler') == 'memcached' && self::is_memcache_configured()) {
+            return true;
+        }
+
+        if (get_config('ssphpsessionhandler') == 'redis' && self::is_redis_configured()) {
+            return true;
+        }
+
         if (empty(get_config('ssphpsessionhandler'))) {
-            return self::is_memcache_configured();
+            // Check Redis
+            $ishandler = self::is_redis_configured();
+            // And check Memcache if no Redis
+            $ishandler = $ishandler ? $ishandler : self::is_memcache_configured();
+            return $ishandler;
         }
 
-        return true;
+        return false;
     }
 
     public static function is_simplesamlphp_installed() {
@@ -679,7 +691,6 @@ class PluginAuthSaml extends PluginAuth {
                 }
             }
         }
-
         return $is_configured;
     }
 
@@ -705,6 +716,43 @@ class PluginAuthSaml extends PluginAuth {
         return $memcache_servers;
     }
 
+    public static function is_redis_configured() {
+        return (bool) PluginAuthSaml::get_redis_master();
+    }
+
+    public static function get_redis_master() {
+        $master = null;
+        if (extension_loaded('redis')) {
+            foreach (self::get_redis_servers() as $server) {
+                if (!empty($server['server']) && !empty($server['mastergroup']) && !empty($server['prefix'])) {
+                    require_once(get_config('libroot') . 'redis/sentinel.php');
+                    $sentinel = new sentinel($server['server']);
+                    $master = $sentinel->get_master_addr($server['mastergroup']);
+                }
+            }
+        }
+        return $master;
+    }
+
+    public static function get_redis_config() {
+        $servers = PluginAuthSaml::get_redis_servers();
+        $master = PluginAuthSaml::get_redis_master();
+        return array('host' => $master->ip,
+                     'port' => (int)$master->port,
+                     'prefix' => $servers[0]['prefix']
+                    );
+    }
+
+    public static function get_redis_servers() {
+        $redissentinelservers = get_config('redissentinelservers');
+        $redismastergroup = get_config('redismastergroup');
+        $redisprefix = get_config('redisprefix');
+        $redis_servers[] = array('server' => $redissentinelservers,
+                                 'mastergroup' => $redismastergroup,
+                                 'prefix' => $redisprefix);
+        return $redis_servers;
+    }
+
     public static function get_idps($xml) {
         $xml = new SimpleXMLElement($xml);
         $xml->registerXPathNamespace('md',   'urn:oasis:names:tc:SAML:2.0:metadata');

htdocs/auth/saml/sp/disco.php

@@ -40,7 +40,7 @@ if (get_field('auth_installed', 'active', 'name', 'saml') != 1) {
     redirect();
 }
 
-if (!extension_loaded('mcrypt')) {
+if (get_config('memcacheservers') && !extension_loaded('mcrypt')) {
     throw new AuthInstanceException(get_string_php_version('errornomcrypt', 'auth.saml'));
 }
 

htdocs/auth/saml/sp/metadata.php

@@ -41,7 +41,7 @@ if (get_field('auth_installed', 'active', 'name', 'saml') != 1) {
     redirect();
 }
 
-if (!extension_loaded('mcrypt')) {
+if (get_config('memcacheservers') && !extension_loaded('mcrypt')) {
     throw new AuthInstanceException(get_string_php_version('errornomcrypt', 'auth.saml'));
 }
 
@@ -288,10 +288,13 @@ if (array_key_exists('output', $_REQUEST) && $_REQUEST['output'] == 'xhtml') {
     $t = new SimpleSAML_XHTML_Template($config, 'metadata.php', 'admin');
 
     $t->data['clipboard.js'] = true;
-    $t->data['header'] = 'saml20-sp';
+    $t->data['available_certs'] = $availableCerts;
+    $t->data['header'] = 'saml20-sp'; // TODO: Replace with headerString in 2.0
+    $t->data['headerString'] = $t->noop('metadata_saml20-idp');
+    $t->data['metaurl'] = get_config('wwwroot') . "auth/saml/sp/metadata.php";
     $t->data['metadata'] = htmlspecialchars($xml);
     $t->data['metadataflat'] = '$metadata[' . var_export($entityId, true) . '] = ' . var_export($metaArray20, true) . ';';
-    $t->data['metaurl'] = get_config('wwwroot') . "auth/saml/sp/metadata.php";
+
     $t->show();
 }
 else {

htdocs/auth/saml/sp/saml1-acs.php

@@ -40,7 +40,7 @@ if (get_field('auth_installed', 'active', 'name', 'saml') != 1) {
     redirect();
 }
 
-if (!extension_loaded('mcrypt')) {
+if (get_config('memcacheservers') && !extension_loaded('mcrypt')) {
     throw new AuthInstanceException(get_string_php_version('errornomcrypt', 'auth.saml'));
 }
 

htdocs/auth/saml/sp/saml2-acs.php

@@ -40,7 +40,7 @@ if (get_field('auth_installed', 'active', 'name', 'saml') != 1) {
     redirect();
 }
 
-if (!extension_loaded('mcrypt')) {
+if (get_config('memcacheservers') && !extension_loaded('mcrypt')) {
     throw new AuthInstanceException(get_string_php_version('errornomcrypt', 'auth.saml'));
 }
 

htdocs/auth/saml/sp/saml2-logout.php

@@ -40,7 +40,7 @@ if (get_field('auth_installed', 'active', 'name', 'saml') != 1) {
     redirect();
 }
 
-if (!extension_loaded('mcrypt')) {
+if (get_config('memcacheservers') && !extension_loaded('mcrypt')) {
     throw new AuthInstanceException(get_string_php_version('errornomcrypt', 'auth.saml'));
 }
 

htdocs/auth/saml/version.php

@@ -11,8 +11,8 @@
 defined('INTERNAL') || die();
 
 $config = new StdClass;
-$config->version = 2017122000;
-$config->release = '1.2.3';
+$config->version = 2018021600;
+$config->release = '1.2.4';
 $config->name = 'saml';
 $config->requires_config = 1;
 $config->requires_parent = 0;

htdocs/auth/session.php

@@ -126,6 +126,32 @@ class Session {
                     Session::create_directory_levels($sessionpath);
                 }
                 break;
+            case 'redis':
+                if (!extension_loaded(get_config('sessionhandler'))) {
+                    throw new ConfigSanityException(get_string('nophpextension', 'error', get_config('sessionhandler')));
+                }
+                else if (
+                         ($redissentinelservers = get_config('redissentinelservers')) &&
+                         ($redismastergroup = get_config('redismastergroup')) &&
+                         ($redisprefix = get_config('redisprefix'))
+                        ) {
+                    require_once(get_config('libroot') . 'redis/sentinel.php');
+
+                    $sentinel = new sentinel($redissentinelservers);
+                    $master = $sentinel->get_master_addr($redismastergroup);
+
+                    if (!empty($master)) {
+                        ini_set('session.save_handler', 'redis');
+                        ini_set('session.save_path', 'tcp://' . $master->ip . ':' . $master->port . '?prefix=' . $redisprefix);
+                    }
+                    else {
+                        throw new ConfigSanityException(get_string('badsessionhandle', 'error', get_config('sessionhandler')));
+                    }
+                }
+                else {
+                    throw new ConfigSanityException(get_string('badsessionhandle', 'error', get_config('sessionhandler')));
+                }
+                break;
             default:
                 throw new ConfigSanityException(get_string('wrongsessionhandle', 'error', get_config('sessionhandler')));
         }

htdocs/lang/en.utf8/error.php

@@ -144,6 +144,7 @@ $string['nopasswordsaltset'] = 'No sitewide password salt has been set. Edit you
 $string['passwordsaltweak'] = 'Your sitewide password salt is not strong enough. Edit your config.php and set the "passwordsaltmain" parameter to a longer secret phrase.';
 $string['urlsecretweak'] = 'The $cfg->urlsecret set for this site has not been changed from the default value. Edit your config.php and set the $cgf->urlsecret parameter to a different string (or null if you do not wish to use a urlsecret).';
 $string['notproductionsite'] = 'This site is not in production mode. Some data may not be available and/or may be out of date.';
+$string['badsessionhandle'] = 'The session save handler "%s" is not configured correctly. Please check the settings in your "config.php" file.';
 $string['wrongsessionhandle'] = 'The session save handler "%s" is not supported in Mahara.';
 $string['nomemcachedserver'] = 'The memcache server "%s" is not reachable. Please check the $cfg->memcacheservers value to make sure it is correct';
 $string['nophpextension'] = 'The PHP extension "%s" is not enabled. Please enable the extension and restart your webserver or choose a different session option.';

htdocs/lib/config-defaults.php

@@ -739,8 +739,8 @@ $cfg->openbadgedisplayer_source = '{"backpack":"https://backpack.openbadges.org/
  * Specify the name of the session handler.
  */
 $cfg->sessionhandler = 'file';
-//$cfg->sessionhandler = 'memcached';
-//$cfg->memcacheservers = 'localhost:11211';
+//$cfg->sessionhandler = 'memcached'; // also set the $cfg->memcacheservers setting if using this one
+//$cfg->sessionhandler = 'redis'; // also set the $cfg->redis* ssettings if using this one
 
 /**
  * @global string $cfg->ssphpsessionhandler
@@ -749,6 +749,14 @@ $cfg->sessionhandler = 'file';
  */
 // $cfg->ssphpsessionhandler = 'memcached';
 
+
+/**
+ * Redis session handling
+ */
+//$cfg->redissentinelservers = "localhost:26379";  // A comma seperated string of hosts:ports
+//$cfg->redismastergroup = 'mymaster';
+//$cfg->redisprefix = 'mahara';
+
 /**
  * @global array $cfg->saml_custommappingfile
  * A list of paths to custom attribute mapping files for SimpleSAMLphp IDP and SP

htdocs/lib/redis/sentinel.php

+<?php
+
+class sentinel {
+
+    private $sentinels = array();
+
+    public $connecttimeout = 1;
+    public $readtimeout = 1;
+    public $persistent = true;
+
+    private $flags;
+
+    private $connected;
+
+    private $socket;
+
+    private $pingonconnect = false;
+
+    public function __construct($sentinels) {
+        if (is_string($sentinels)) {
+            $sentinels = explode(',', $sentinels);
+        }
+        $this->sentinels = $sentinels;
+
+        $this->flags = STREAM_CLIENT_CONNECT;
+
+        $this->connected = false;
+
+    }
+
+    public function __destruct() {
+        if (!$this->persistent && $this->connected) {
+            $this->disconnect();
+        }
+    }
+
+    public function connecttopool() {
+        if ($this->connected) {
+            return true;
+        }
+
+        foreach ($this->sentinels as $sentinel) {
+            if ($this->connect($sentinel)) {
+                return true;
+            }
+        }
+
+        throw new \Exception('Unable to connect to sentinel pool');
+    }
+
+
+    private function connect($sentinel) {
+
+        if ($this->persistent) {
+            $this->socket = @stream_socket_client($sentinel, $errorno, $errstr, $this->connecttimeout, STREAM_CLIENT_CONNECT | STREAM_CLIENT_PERSISTENT);
+        }
+        else {
+            $this->socket = @stream_socket_client($sentinel, $errorno, $errstr, $this->connecttimeout);
+        }
+
+        if (!$this->socket) {
+            $this->connected = false;
+            return false;
+        }
+
+        $this->connected = true;
+
+        stream_set_blocking($this->socket, true);
+        stream_set_timeout($this->socket, $this->readtimeout);
+
+        // Test sentinel is alive
+        if ($this->pingonconnect) {
+            fwrite($this->socket, "PING\n");
+            if (trim(fgets($this->socket)) != '+PONG') {
+                fclose($this->socket);
+                $this->connected = false;
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public function disconnect() {
+        fclose($this->socket);
+        $this->connected = false;
+    }
+
+    public function get_master_addr($name) {
+
+        $cmd = "get-master-addr-by-name $name";
+
+        $this->command($cmd);
+        if (!$resp = $this->readreply()) {
+            return false;
+        }
+
+        $ret = new \stdClass();
+        $ret->ip = $resp[0];
+        $ret->port = $resp[1];
+
+        return ($ret);
+    }
+
+
+    private function command($command) {
+        if (!$this->connected) {
+            $this->connecttopool();
+        }
+
+        if (!$this->connected) {
+            return false;
+        }
+        $cmd = "SENTINEL $command\n";
+
+        $cmdlen = strlen($cmd);
+        $lastwrite = 0;
+        for ($written = 0; $written < $cmdlen; $written += $lastwrite) {
+            $lastwrite = fwrite($this->socket, substr($cmd, $written));
+
+            if ($lastwrite === false || $lastwrite == 0) {
+                $this->connected = false;
+                throw new \Exception('Failed to write command to stream');
+            }
+        }
+    }
+
+
+    private function readreply() {
+        if (!$this->connected) {
+            return false;
+        }
+
+        $resp = fgets($this->socket);
+
+        $type = substr($resp, 0, 1);
+
+        switch($type) {
+
+            // Error response
+            case '-':
+                throw new \Exception('Error response received: '.$resp);
+                break;
+
+            // In-line response
+            case '+':
+                $response = substr($resp, 1);
+                return(substr($resp, 1));
+
+            // Defined size response
+            case '$':
+                $size = (int) substr($resp, 1);
+                $resp = stream_get_contents($this->socket, $size + 2);
+                if ($resp === false) {
+                    throw new \Exception('Failed to read from stream');
+                }
+                return (trim($resp));
+
+            // Int response
+            case ':':
+                return ((int)substr($reply, 1));
+
+            // Multi line response
+            case '*':
+                $multireponse = array();
+                $size = (int) substr($resp, 1);
+
+                for ($i = 0; $i < $size; $i++) {
+                    $multireponse[] = $this->readreply();
+                }
+                return($multireponse);
+
+            // Unknown response.
+            default:
+                throw new \Exception('Unknown read response from stream');
+        }
+    }
+}
+<