Commit deff8a39 authored by Stuart McDonald's avatar Stuart McDonald Committed by Stuart McDonald

(#3014) Proxy server support & (#3019) Remove Snoopy

parent cb95bdf5
......@@ -90,6 +90,34 @@ $networkingform = pieform(
'defaultvalue' => get_config('promiscuousmode'),
'options' => $yesno,
'proxyfieldset' => array(
'type' => 'fieldset',
'legend' => get_string('proxylegend', 'admin'),
'elements' => array(
'proxyaddress' => array(
'type' => 'text',
'title' => get_string('proxyaddress', 'admin'),
'description' => get_string('proxyaddressdescription', 'admin'),
'defaultvalue' => get_config('proxyaddress'),
'proxyauthmodel' => array(
'type' => 'select',
'title' => get_string('proxyauthmodel', 'admin'),
'description' => get_string('proxyauthmodeldescription', 'admin'),
'defaultvalue' => get_config('proxyauthmodel'),
'options' => array(
'' => 'None',
'basic' => 'Basic (NCSA)',
'proxyauthcredentials' => array(
'type' => 'text',
'title' => get_string('proxyauthcredentials', 'admin'),
'description' => get_string('proxyauthcredentialsdescription', 'admin'),
'defaultvalue' => get_config('proxyauthcredentials'),
'submit' => array(
'type' => 'submit',
'value' => get_string('savechanges','admin')
......@@ -136,6 +164,33 @@ function networkingform_submit(Pieform $form, $values) {
if(get_config('proxyaddress') != $values['proxyaddress']) {
if(!set_config('proxyaddress', $values['proxyaddress'])) {
else {
$reply .= get_string('proxyaddressset', 'admin');
if(get_config('proxyauthmodel') != $values['proxyauthmodel']) {
if(!set_config('proxyauthmodel', $values['proxyauthmodel'])) {
else {
$reply .= get_string('proxyauthmodelset', 'admin');
if(get_config('proxyauthcredentials') != $values['proxyauthcredentials']) {
if(!set_config('proxyauthcredentials', $values['proxyauthcredentials'])) {
else {
$reply .= get_string('proxyauthcredntialsset', 'admin');
$form->reply(PIEFORM_OK, array(
'message' => ($reply == '') ? get_string('networkingunchanged','admin') : $reply,
'goto' => '/admin/site/networking.php',
......@@ -532,21 +532,22 @@ function get_public_key($uri, $application=null) {
$wwwroot = dropslash(get_config('wwwroot'));
$rq = xmlrpc_encode_request('system/keyswap', array($wwwroot, $openssl->certificate), array("encoding" => "utf-8"));
$ch = curl_init($uri . $xmlrpcserverurl);
curl_setopt($ch, CURLOPT_TIMEOUT, 60);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_USERAGENT, 'Moodle');
curl_setopt($ch, CURLOPT_POSTFIELDS, $rq);
curl_setopt($ch, CURLOPT_HTTPHEADER, array("Content-Type: text/xml charset=UTF-8", 'Expect: '));
$config = array(
CURLOPT_URL => $uri . $xmlrpcserverurl,
CURLOPT_HTTPHEADER => array("Content-Type: text/xml charset=UTF-8", 'Expect: '),
$result = http_request($config);
$raw = curl_exec($ch);
if (empty($raw)) {
if (empty($result->data)) {
throw new XmlrpcClientException('CURL connection failed');
$response_code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$response_code = $result->info['http_code'];
$response_code_prefix = substr($response_code, 0, 1);
if ('2' != $response_code_prefix) {
......@@ -557,7 +558,7 @@ function get_public_key($uri, $application=null) {
$res = xmlrpc_decode($raw);
$res = xmlrpc_decode($result->data);
// XMLRPC error messages are returned as an array
......@@ -38,7 +38,7 @@ require_once(get_config('docroot') . 'api/xmlrpc/lib.php');
class AuthXmlrpc extends Auth {
* Get the party started with an optional id
* TODO: appraise
* @param int $id The auth instance id
......@@ -259,24 +259,20 @@ class PluginBlocktypeExternalfeed extends SystemBlocktype {
$snoopy = new Snoopy();
$snoopy->curl_path = '/usr/bin/curl'; // TODO: make configurable later. This is the default for debian
$config = array(CURLOPT_URL => $source);
// This is to disable warnings from snoopy while it performs its work.
// By all means, comment out while debugging any problems
$oldlevel = error_reporting(0);
$result = $snoopy->fetch($source);
$result = http_request($config);
if (!$result) {
$cache[$source] = new XML_Feed_Parser_Exception($snoopy->error);
throw $cache[$source];
if($result->data) {
if ($result->error) {
$cache[$source] = $result->error;
throw $cache[$source];
try {
$feed = new XML_Feed_Parser($snoopy->results, false, true, false);
$feed = new XML_Feed_Parser($result->data, false, true, false);
catch (XML_Feed_Parser_Exception $e) {
$cache[$source] = $e;
......@@ -208,6 +208,17 @@ $string['promiscuousmodeenabled'] = 'Auto-register has been enabled. ';
$string['promiscuousmodedescription'] = 'Create an institution record for any host that connects to you, and allow its users to log on to Mahara';
$string['wwwroot'] = 'WWW Root';
$string['wwwrootdescription'] = 'This is the URL at which your users access this Mahara installation, and the URL the SSL keys are generated for';
$string['proxylegend'] = 'Proxy settings';
$string['proxyaddress'] = 'Proxy address';
$string['proxyaddressdescription'] = 'If your site uses a proxy server to access the internet, specify the proxies in <em>hostname:portnumber</em> notation';
$string['proxyaddressset'] = 'Proxy address set';
$string['proxyauthmodel'] = 'Proxy authenticated model';
$string['proxyauthmodeldescription'] = 'Select your proxy\'s authentication model, if appropriate';
$string['proxyauthmodelset'] = 'Proxy authentication model has been set';
$string['proxyauthcredentials'] = 'Proxy credentials';
$string['proxyauthcredentialsdescription'] = 'Enter the credentials required for your proxy to authenticate your web server in <em>username:password</em> format';
$string['proxyauthcredntialsset'] = 'Proxy authentication credentials set';
// Upload CSV
$string['csvfile'] = 'CSV File';
Monte Ohrt <>
- main Snoopy work
Andrei Zmievski <>
- miscellaneous fixes
- read timeout support
- file submission capability
Gene Wood <>
- bug fixes
- security fixes
This diff is collapsed.
Version 1.2.4
- fix command line escapement vulnerability with execution of curl binary on https fetches (mohrt)
Version 1.2.3
- updated the version variable in the code to reflect the new version number
- fixed a typo that I introduced in 1.2.2 (the first character of the file is a "z" (gene_wood, Marc Desrousseaux, Jan Pedersen)
- fixed BUG # 1328793 : fetch is case sensetive when it comes to the scheme (http / https) (gene_wood)
Version 1.2.2
- incorporated PATCH # 985470 : pass port information in http 1.1 Host header ( ) (gene_wood)
- fixed BUG # 1110049 : redirect is case sensitive
- fixed bug in security bugfix from 1.2.1 (gene_wood, kellan, zaruba)
Version 1.2.1
- fixed potential security issue with unchecked variables being passed to exec (for https with curl) (gene_wood)
- fixed BUG # 1086830 : submitlinks,fetchlinks and submittext expandlinks with the URI of the original page not the refreshed page (gene_wood)
- fixed BUG # 1077870 : Snoopy can't deal with multiple spaces in a refresh tag (gene_wood)
- fixed BUG # 864047 : Root relative links are treated as relative (gene_wood)
- fixed BUG # 1097134 : Undefined URI_PARTS["path"] generates Notice (gene_wood)
Version 1.2
- fixed BUG # 1014823 : Meta redirect regex inaccurate (gene_wood)
- fixed BUG # 999079 : Trailing slashes not removed in uri passed to fetchlinks (gene_wood)
- fixed BUG # 642958 and 912060 : $URI_PARTS["query"] causing undefined variable notices (gene_wood)
- fixed BUG # 626849 : cURL security risk (Tajh Leitso, gene_wood)
- fixed BUG # 626849 : Corrects the redirect function under the submit functions (Tajh Leitso, gene_wood)
- fixed BUG # 912060 : Undefined variable: postdata (gene_wood)
- fixed BUG # 858526 : win32 tmp/$headerfile create error (gene_wood)
- fixed BUG # 929682 : Called undefined function is_executable() on line 194. (gene_wood)
- fixed BUG # 859711 : typo: (gene_wood)
- fixed BUG # 852993 : double urlencoding breaks redirect (gene_wood)
- added proxy user/pass support (Robert Zwink, Monte)
- fixed post data array problem (stefan, Monte)
Version 1.01
- fixed problem with PHP 4.3.2 and fread() (Monte)
Version 1.0
- added textarea to stripform functionality (Monte)
- fixed multiple cookie setting problem (Monte)
- fixed problem where extra text inside <frame src (Monte)
- fixed problem where extra text inside <a href (Monte)
- removed http request header from curl fetched
documents, not needed (Monte)
- added carriage return to newlines on headers (Monte)
- fixed bug with curl, removed single quotes
- fixed bug with curl and "&" in the URL
- added ability to post files. (Andrei)
Version 0.94
- Added fetchform() function
- Fixed misc issues with frames
- Added SSL support via cURL
- fixed bug with posting arrays of data
- added status variable for http status
Version 0.93
- fixed bug with hostname match in a redirect location header
- added $lastredirectaddr variable
Version 0.92
- fixed redirect bug with MS web server
- added ability to pass set cookies through redirects
- added ability to traverse html frames
Version 0.91
- fixed bug with return headers being overwritten.
Please read the NEWS file for important notes. (Monte)
Version 0.9
- added support for read timeouts (Andrei)
- standardized distribution (Andrei)
Version 0.1e
- fixed bug in fetchlinks logic (Monte)
Version 0.1d
- fixed redirect bug without fully qualified url (Monte)
Version 0.1c
- fixed bug on submitting formvars after a redirect (Monte)
Version 0.1b
- fixed bug to allow empty post vars on a submit (Monte)
Version 0.1
- initial release (Monte)
Q: Why can't I fetch https pages?
A: Using Snoopy to fetch an https page requires curl. Check if curl is installed on your host. If curl is installed, it may be located in a different place than the default. By default Snoopy looks for curl in /usr/local/bin/curl. Run 'which curl' and find out your location. If it differs from the default, then you'll need to set the $snoopy->curl_path variable to the location of your curl installation. Here's an example of the code :
include "Snoopy.class.php";
$snoopy = new Snoopy;
Q: where does the function preg_match_all come from?
A: PCRE functions in PHP 3.0.9 and later
Q: I get the error: Warning: Wrong parameter count for fsockopen()
A: Upgrade your verion of PHP to 3.0.9 or later
Q: Snoopy cuts of my results every time. What's wrong?
A: Upgrade your verion of PHP to 3.0.9 or later
Put Snoopy.class.php into one of the directories specified in your
php.ini include_path directive.
October 22, 2008
https fetches were not properly escaping shell args for curl binary execution. This is fixed.
November 7, 2005
A typo was introduced in 1.2.2 which broke the whole release. This has been fixed.
A couple small fixes have been implemented also.
October 30, 2005
Fixed a bug with the bugfix for the security hole.
October 24, 2005
Fixed a few outstanding bugs and a potential security hole.
November 17, 2004
Fixed a number of outstanding bugs.
PHP fixed a bug with fread() which consequently broke the way Snoopy called it. This has been fixed.
Renamed to Snoopy.class.php for proper file extention.
Added fetchform() function for fetching form elements from an html page.
For SSL support, you must have cURL installed. see
for details. Snoopy does not use the cURL library fuctions within PHP,
as these are not stable as of this Snoopy release.
Fixed bug with posting arrays of data.
Added status variable to track http status.
Several other bug fixes, see Changelog.
A bug was fixed with redirection headers not containing the hostname, doubling up the redirection location URL.
There is also a new variable, $lastredirectaddr that contains the last redirection URL.
March 9, 2000
A bug was fixed with redirection on MS web servers. Also, cookies are now passed through redirects.
This release also adds the ability to traverse html framed pages. Just set $maxframes to the recursion depth you want to allow, and results are returned in $this->results as an array. See the README for an example.
February 22, 2000
In previous versions of Snoopy, $this->header was an array containing key/value pairs of headers returned from fetched content, not including HTTP and GET headers. If a key value was the same, the old value was overwritten (Two Set-Cookie: headers for example). This was overcome by making $this->header a simple array containing every header returned. Therefore, it will now be up to the programmer to split these headers into key/value pairs if so desired.
Snoopy - the PHP net client v1.2.4
include "Snoopy.class.php";
$snoopy = new Snoopy;
print $snoopy->results;
print $snoopy->results;
$submit_url = "";
$submit_vars["q"] = "amiga";
$submit_vars["submit"] = "Search!";
$submit_vars["searchhost"] = "Altavista";
print $snoopy->results;
echo "<PRE>\n";
echo htmlentities($snoopy->results[0]);
echo htmlentities($snoopy->results[1]);
echo htmlentities($snoopy->results[2]);
echo "</PRE>\n";
print $snoopy->results;
What is Snoopy?
Snoopy is a PHP class that simulates a web browser. It automates the
task of retrieving web page content and posting forms, for example.
Some of Snoopy's features:
* easily fetch the contents of a web page
* easily fetch the text from a web page (strip html tags)
* easily fetch the the links from a web page
* supports proxy hosts
* supports basic user/pass authentication
* supports setting user_agent, referer, cookies and header content
* supports browser redirects, and controlled depth of redirects
* expands fetched links to fully qualified URLs (default)
* easily submit form data and retrieve the results
* supports following html frames (added v0.92)
* supports passing cookies on redirects (added v0.92)
Snoopy requires PHP with PCRE (Perl Compatible Regular Expressions),
which should be PHP 3.0.9 and up. For read timeout support, it requires
PHP 4 Beta 4 or later. Snoopy was developed and tested with PHP 3.0.12.
This is the method used for fetching the contents of a web page.
$URI is the fully qualified URL of the page to fetch.
The results of the fetch are stored in $this->results.
If you are fetching frames, then $this->results
contains each frame fetched in an array.
This behaves exactly like fetch() except that it only returns
the text from the page, stripping out html tags and other
irrelevant data.
This behaves exactly like fetch() except that it only returns
the form elements from the page, stripping out html tags and other
irrelevant data.
This behaves exactly like fetch() except that it only returns
the links from the page. By default, relative links are
converted to their fully qualified URL form.
This submits a form to the specified $URI. $formvars is an
array of the form variables to pass.
This behaves exactly like submit() except that it only returns
the text from the page, stripping out html tags and other
irrelevant data.
This behaves exactly like submit() except that it only returns
the links from the page. By default, relative links are
converted to their fully qualified URL form.
CLASS VARIABLES: (default value in parenthesis)
$host the host to connect to
$port the port to connect to
$proxy_host the proxy host to use, if any
$proxy_port the proxy port to use, if any
$agent the user agent to masqerade as (Snoopy v0.1)
$referer referer information to pass, if any
$cookies cookies to pass if any
$rawheaders other header info to pass, if any
$maxredirs maximum redirects to allow. 0=none allowed. (5)
$offsiteok whether or not to allow redirects off-site. (true)
$expandlinks whether or not to expand links to fully qualified URLs (true)
$user authentication username, if any
$pass authentication password, if any
$accept http accept types (image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*)
$error where errors are sent, if any
$response_code responde code returned from server
$headers headers returned from server
$maxlength max return data length
$read_timeout timeout on read operations (requires PHP 4 Beta 4+)
set to 0 to disallow timeouts
$timed_out true if a read operation timed out (requires PHP 4 Beta 4+)
$maxframes number of frames we will follow
$status http status of fetch
$temp_dir temp directory that the webserver can write to. (/tmp)
$curl_path system path to cURL binary, set to false if none
Example: fetch a web page and display the return headers and
the contents of the page (html-escaped):
include "Snoopy.class.php";
$snoopy = new Snoopy;
$snoopy->user = "joe";
$snoopy->pass = "bloe";
echo "response code: ".$snoopy->response_code."<br>\n";
while(list($key,$val) = each($snoopy->headers))
echo $key.": ".$val."<br>\n";
echo "<p>\n";
echo "<PRE>".htmlspecialchars($snoopy->results)."</PRE>\n";
echo "error fetching document: ".$snoopy->error."\n";
Example: submit a form and print out the result headers
and html-escaped page:
include "Snoopy.class.php";
$snoopy = new Snoopy;
$submit_url = "";
$submit_vars["q"] = "amiga";
$submit_vars["submit"] = "Search!";
$submit_vars["searchhost"] = "Altavista";
while(list($key,$val) = each($snoopy->headers))
echo $key.": ".$val."<br>\n";
echo "<p>\n";
echo "<PRE>".htmlspecialchars($snoopy->results)."</PRE>\n";
echo "error fetching document: ".$snoopy->error."\n";
Example: showing functionality of all the variables:
include "Snoopy.class.php";
$snoopy = new Snoopy;
$snoopy->proxy_host = "";
$snoopy->proxy_port = "8080";
$snoopy->agent = "(compatible; MSIE 4.01; MSN 2.5; AOL 4.0; Windows 98)";
$snoopy->referer = "";
$snoopy->cookies["SessionID"] = 238472834723489l;
$snoopy->cookies["favoriteColor"] = "RED";
$snoopy->rawheaders["Pragma"] = "no-cache";
$snoopy->maxredirs = 2;
$snoopy->offsiteok = false;
$snoopy->expandlinks = false;
$snoopy->user = "joe";
$snoopy->pass = "bloe";
while(list($key,$val) = each($snoopy->headers))
echo $key.": ".$val."<br>\n";
echo "<p>\n";
echo "<PRE>".htmlspecialchars($snoopy->results)."</PRE>\n";
echo "error fetching document: ".$snoopy->error."\n";
Example: fetched framed content and display the results
include "Snoopy.class.php";
$snoopy = new Snoopy;
$snoopy->maxframes = 5;
echo "<PRE>".htmlspecialchars($snoopy->results[0])."</PRE>\n";
echo "<PRE>".htmlspecialchars($snoopy->results[1])."</PRE>\n";
echo "<PRE>".htmlspecialchars($snoopy->results[2])."</PRE>\n";
echo "error fetching document: ".$snoopy->error."\n";
Copyright(c) 1999,2000 ispi. All rights reserved.
This software is released under the GNU General Public License.
Please read the disclaimer at the top of the Snoopy.class.php file.
Special Thanks to:
Peter Sorger <> help fixing a redirect bug
Andrei Zmievski <> implementing time out functionality
Patric Sandelin <> help with fetchform debugging
Carmelo <> misc bug fixes with frames
This diff is collapsed.
* fetch other types of protocols such as ftp, nntp, gopher, etc.
* post forms with http file upload (I didn't have this need,
but it should be fairly straightforward)
* expand links, image tags, and form actions to fully
qualified URLs
* none known
......@@ -2554,4 +2554,33 @@ function build_pagination_pagelink($class, $url, $offset, $text, $title, $disabl
return $return;
function http_request($config) {
$ch = curl_init();
// standard curl_setopt stuff; configs passed to the function can override these
curl_setopt($ch, CURLOPT_TIMEOUT, 60);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt_array($ch, $config);
if($proxy_address = get_config('proxyaddress')) {
curl_setopt($ch, CURLOPT_PROXY, $proxy_address);