BehatHooks.php 19.5 KB
Newer Older
Son Nguyen's avatar
Son Nguyen committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
<?php
/**
 * @package    mahara
 * @subpackage test/behat
 * @author     Son Nguyen, Catalyst IT Ltd
 * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL version 3 or later
 * @copyright  For copyright information on Mahara, please see the README file distributed with this software.
 * @copyright  portions from mahara Behat, 2013 David Monllaó
 *
 */

/**
 * Behat accepts hooks after and before each
 * suite, feature, scenario and step.
 *
 * This methods are used by Behat CLI command.
 *
 */


Son Nguyen's avatar
Son Nguyen committed
21
require_once(__DIR__ . '/BehatBase.php');
Son Nguyen's avatar
Son Nguyen committed
22
23
24
25
26
27
28
29
30
31
32
33

use Behat\Behat\Event\SuiteEvent as SuiteEvent,
    Behat\Behat\Event\FeatureEvent as FeatureEvent,
    Behat\Behat\Event\ScenarioEvent as ScenarioEvent,
    Behat\Behat\Event\StepEvent as StepEvent,
    Behat\Mink\Exception\DriverException as DriverException,
    WebDriver\Exception\NoSuchWindow as NoSuchWindow,
    WebDriver\Exception\UnexpectedAlertOpen as UnexpectedAlertOpen,
    WebDriver\Exception\UnknownError as UnknownError,
    WebDriver\Exception\CurlExec as CurlExec,
    WebDriver\Exception\NoAlertOpenError as NoAlertOpenError;

Son Nguyen's avatar
Son Nguyen committed
34
35
use Behat\Behat\Hook\Scope\BeforeStepScope;
use Behat\Behat\Hook\Scope\AfterStepScope;
Son Nguyen's avatar
Son Nguyen committed
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
/**
 * Hooks to the behat process.
 *
 * Implement hooks after and before each
 * suite, feature, scenario and step for Mahara
 *
 * Throws generic Exception because they are captured by Behat.
 *
 */
class BehatHooks extends BehatBase {

    /**
     * @var For actions that should only run once.
     */
    protected static $initprocessesfinished = false;

    /**
     * Some exceptions can only be caught in a before or after step hook,
     * they can not be thrown there as they will provoke a framework level
     * failure, but we can store them here to fail the step in i_look_for_exceptions()
     * which result will be parsed by the framework as the last step result.
     *
     * @var Null or the exception last step throw in the before or after hook.
     */
    protected static $currentstepexception = null;

    /**
     * If we are saving any kind of dump on failure we should use the same parent dir during a run.
     *
     * @var The parent dir name
     */
    protected static $faildumpdirname = false;

    /**
Son Nguyen's avatar
Son Nguyen committed
70
     * Make sure the test site is installed and enabled for behat tests.
Son Nguyen's avatar
Son Nguyen committed
71
72
73
74
75
76
     *
     * @static
     * @throws Exception
     * @BeforeSuite
     */
    public static function before_suite($event) {
Son Nguyen's avatar
Son Nguyen committed
77
        global $CFG, $db, $SESSION, $USER, $THEME;
Son Nguyen's avatar
Son Nguyen committed
78
79
80
81

        // Defined only when the behat CLI command is running, the mahara init setup process will
        // read this value and switch to $CFG->behat_dataroot and $CFG->behat_dbprefix instead of
        // the normal site.
Son Nguyen's avatar
Son Nguyen committed
82
        define('BEHAT_UTIL', 1);
Son Nguyen's avatar
Son Nguyen committed
83

Son Nguyen's avatar
Son Nguyen committed
84
        define('INTERNAL', 1);
Son Nguyen's avatar
Son Nguyen committed
85
        define('CLI', 1);
Son Nguyen's avatar
Son Nguyen committed
86
87
88
89
90

        // With BEHAT_TEST we will be using $CFG->behat_* instead of $CFG->dataroot, $CFG->dbprefix and $CFG->wwwroot.
        require_once(dirname(dirname(dirname(dirname(__DIR__)))) . '/init.php');

        // Now that we are in Mahara env.
Son Nguyen's avatar
Son Nguyen committed
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
        require_once('upgrade.php');
        require_once('file.php');
        require_once(dirname(dirname(dirname(__DIR__))) . '/classes/TestLock.php');
        require_once(__DIR__ . '/util.php');

        // Initialize and enable the test site if possible
        $statuscode = BehatTestingUtil::get_test_env_status();
        switch ($statuscode) {
            case BEHAT_MAHARA_EXITCODE_OUTOFDATEDB:
                BehatTestingUtil::drop_site();
            case BEHAT_MAHARA_EXITCODE_NOTINSTALLED:
                BehatTestingUtil::install_site();
            case BEHAT_MAHARA_EXITCODE_NOTENABLED:
                BehatTestingUtil::start_test_mode();
            case 0:
                break;
            default:
                throw new Exception($statuscode.'The test site is not ready to test.
    Please run php ' . dirname(dirname(dirname(dirname(__DIR__)))) . 'testing/frameworks/behat/cli/init.php to initialize the test site');;
            break;
Son Nguyen's avatar
Son Nguyen committed
111
112
        }

Son Nguyen's avatar
Son Nguyen committed
113
114
        if (!empty($CFG->behat_faildump_path) && !is_writable($CFG->behat_faildump_path)) {
            throw new Exception('You set $CFG->behat_faildump_path to a non-writable directory');
Son Nguyen's avatar
Son Nguyen committed
115
        }
Son Nguyen's avatar
Son Nguyen committed
116
    }
Son Nguyen's avatar
Son Nguyen committed
117

Son Nguyen's avatar
Son Nguyen committed
118
119
120
121
122
123
124
125
126
    /**
     * Clean test database and dataroot and disable the test environment.
     *
     * @static
     * @throws Exception
     * @AfterSuite
     */
    public static function after_suite($event) {
        global $CFG, $db, $SESSION, $USER, $THEME;
Son Nguyen's avatar
Son Nguyen committed
127

Son Nguyen's avatar
Son Nguyen committed
128
129
130
        // Check if the test environment is ready: dataroot, database, server
        if (!defined('BEHAT_TEST')) {
            throw new Exception('The test site is not enabled for behat testing');
Son Nguyen's avatar
Son Nguyen committed
131
        }
Son Nguyen's avatar
Son Nguyen committed
132
133
134

        //BehatTestingUtil::drop_site();
        BehatTestingUtil::stop_test_mode();
Son Nguyen's avatar
Son Nguyen committed
135
136
137
138
139
140
141
142
143
144
145
    }

    /**
     * Resets the test environment.
     *
     * @throws Exception If here we are not using the test database it should be because of a coding error
     * @BeforeScenario
     */
    public function before_scenario($event) {
        global $CFG;

Son Nguyen's avatar
Son Nguyen committed
146
147
148
         // Check if the test environment is ready: dataroot, database, server
        if (!defined('BEHAT_TEST')) {
            throw new Exception('The test site is not enabled for behat testing');
Son Nguyen's avatar
Son Nguyen committed
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
        }

        // Check if the browser is running and supports javascript
        $moreinfo = 'More info in ' . BehatCommand::DOCS_URL . '#Running_tests';
        $driverexceptionmsg = 'Selenium server is not running, you need to start it to run tests that involve Javascript. ' . $moreinfo;
        try {
            $session = $this->getSession();
        }
        catch (CurlExec $e) {
            // Exception thrown by WebDriver, so only @javascript tests will be caugth; in
            throw new Exception($driverexceptionmsg);
        }
        catch (DriverException $e) {
            throw new Exception($driverexceptionmsg);
        }
        catch (UnknownError $e) {
            // Generic 'I have no idea' Selenium error. Custom exception to provide more feedback about possible solutions.
            throw new Exception($e);
        }

        // Register the named selectors for mahara
        if (self::is_first_scenario()) {
            BehatSelectors::register_mahara_selectors($session);
            BehatContextHelper::set_session($session);
            // Reset the browser
            $session->restart();
            // Run all test with medium (1024x768) screen size, to avoid responsive problems.
            $this->resize_window('medium');
        }

        // Reset $SESSION.
        $_SESSION = array();
        $SESSION = new stdClass();
        $_SESSION['SESSION'] =& $SESSION;

        BehatTestingUtil::reset_database();
        BehatTestingUtil::reset_dataroot();

        // Reset the nasty strings list used during the last test.
Son Nguyen's avatar
Son Nguyen committed
188
        //NastyStrings::reset_used_strings();
Son Nguyen's avatar
Son Nguyen committed
189

Son Nguyen's avatar
Son Nguyen committed
190
191
        // Set current user is admin

Son Nguyen's avatar
Son Nguyen committed
192
193
194
195
196
197
        // Start always in the the homepage.
        try {
            // Let's be conservative as we never know when new upstream issues will affect us.
            $session->visit($this->locate_path('/'));
        }
        catch (UnknownError $e) {
Son Nguyen's avatar
Son Nguyen committed
198
            throw new Exception($e);
Son Nguyen's avatar
Son Nguyen committed
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
        }

        // Checking that the root path is a mahara test site.
        if (!self::$initprocessesfinished) {
            $notestsiteexception = new Exception('The base URL (' . $CFG->wwwroot . ') is not a behat test site, ' .
                'ensure you started the built-in web server in the correct directory or your web server is correctly started and set up');
            $this->find("xpath", "//head/child::title[contains(., '" . BehatTestingUtil::BEHATSITENAME . "')]", $notestsiteexception);

            self::$initprocessesfinished = true;
        }
    }

    /**
     * Wait for JS to complete before beginning interacting with the DOM.
     *
     * Executed only when running against a real browser. We wrap it
     * all in a try & catch to forward the exception to i_look_for_exceptions
     * so the exception will be at scenario level, which causes a failure, by
     * default would be at framework level, which will stop the execution of
     * the run.
     *
Son Nguyen's avatar
Son Nguyen committed
220
     * @BeforeStep
Son Nguyen's avatar
Son Nguyen committed
221
     */
Son Nguyen's avatar
Son Nguyen committed
222
223
224
225
226
227
228
229
230
231
232
233
//     public function before_step(BeforeStepScope $scope) {

//         if ($this->running_javascript()) {
//             try {
//                 $this->wait_for_pending_js();
//                 self::$currentstepexception = null;
//             }
//             catch (Exception $e) {
//                 self::$currentstepexception = $e;
//             }
//         }
//     }
Son Nguyen's avatar
Son Nguyen committed
234

235
236
237
238
239
240
241
242
243
244
    /**
     * @BeforeScenario
     *
     * @param BeforeScenarioScope $scope
     *
     */
    public function setUpTestEnvironment($scope) {
        $this->currentScenario = $scope->getScenario();
    }

Son Nguyen's avatar
Son Nguyen committed
245
246
247
248
249
250
251
252
253
254
255
256
    /**
     * Wait for JS to complete after finishing the step.
     *
     * With this we ensure that there are not AJAX calls
     * still in progress.
     *
     * Executed only when running against a real browser. We wrap it
     * all in a try & catch to forward the exception to i_look_for_exceptions
     * so the exception will be at scenario level, which causes a failure, by
     * default would be at framework level, which will stop the execution of
     * the run.
     *
Son Nguyen's avatar
Son Nguyen committed
257
     * Take screenshot if the step failed
Son Nguyen's avatar
Son Nguyen committed
258
259
260
261
262
     *
     * This includes creating an HTML dump of the content if there was a failure.
     *
     * @AfterStep
     */
Son Nguyen's avatar
Son Nguyen committed
263
    public function after_step(AfterStepScope $scope) {
Son Nguyen's avatar
Son Nguyen committed
264
265
        global $CFG;

Son Nguyen's avatar
Son Nguyen committed
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
        if ($this->running_javascript()) {
//            && in_array($scope->getStep()->getKeywordType(), array('Given', 'When'))) {
            try {
                $this->wait_for_pending_js();
                self::$currentstepexception = null;
            }
            catch (UnexpectedAlertOpen $e) {
                self::$currentstepexception = $e;

                // Accepting the alert so the framework can continue properly running
                // the following scenarios. Some browsers already closes the alert, so
                // wrapping in a try & catch.
                try {
                    $this->getSession()->getDriver()->getWebDriverSession()->accept_alert();
                }
                catch (Exception $e) {
                    // Catching the generic one as we never know how drivers reacts here.
                }
            }
            catch (Exception $e) {
                self::$currentstepexception = $e;
            }
        }
Son Nguyen's avatar
Son Nguyen committed
289
        if (!empty($CFG->behat_faildump_path) &&
Son Nguyen's avatar
Son Nguyen committed
290
291
                $scope->getTestResult()->getResultCode() === 99) {
            $this->take_contentdump($scope);
Son Nguyen's avatar
Son Nguyen committed
292
        }
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310

        // if test has failed, and is not an api test, get screenshot

        if (!$scope->getTestResult()->isPassed()) {
            // create filename string

            $featureFolder = preg_replace('/\W/', '', $scope->getFeature()->getTitle());
            $scenarioName = $this->currentScenario->getTitle();
            $fileName = preg_replace('/\W/', '', $scenarioName) . '.png';

            // create screenshots directory if it doesn't exist
            if (!file_exists($CFG->behat_dataroot . '/behat/html_results/screenshots/' . $featureFolder)) {
                mkdir($CFG->behat_dataroot . '/behat/html_results/screenshots/' . $featureFolder, $CFG->directorypermissions, true);
            }

            // For Selenium2 Driver you can use:
            file_put_contents($CFG->behat_dataroot . '/behat/html_results/screenshots/' . $featureFolder . '/' . $fileName, $this->getSession()->getDriver()->getScreenshot());
        }
Son Nguyen's avatar
Son Nguyen committed
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
    }

    /**
     * Getter for self::$faildumpdirname
     *
     * @return string
     */
    protected function get_run_faildump_dir() {
        return self::$faildumpdirname;
    }

    /**
     * Take screenshot when a step fails.
     *
     * @throws Exception
Son Nguyen's avatar
Son Nguyen committed
326
     * @param AfterStepScope $scope
Son Nguyen's avatar
Son Nguyen committed
327
     */
Son Nguyen's avatar
Son Nguyen committed
328
    protected function take_screenshot(AfterStepScope $scope) {
Son Nguyen's avatar
Son Nguyen committed
329
330
331
332
333
        // Goutte can't save screenshots.
        if (!$this->running_javascript()) {
            return false;
        }

Son Nguyen's avatar
Son Nguyen committed
334
        list ($dir, $filename) = $this->get_faildump_filename($scope, 'png');
Son Nguyen's avatar
Son Nguyen committed
335
336
337
338
339
340
341
        $this->saveScreenshot($filename, $dir);
    }

    /**
     * Take a dump of the page content when a step fails.
     *
     * @throws Exception
Son Nguyen's avatar
Son Nguyen committed
342
     * @param AfterStepScope $scope
Son Nguyen's avatar
Son Nguyen committed
343
     */
Son Nguyen's avatar
Son Nguyen committed
344
345
    protected function take_contentdump(AfterStepScope $scope) {
        list ($dir, $filename) = $this->get_faildump_filename($scope, 'html');
Son Nguyen's avatar
Son Nguyen committed
346
347
348
349
350
351
352
353
354
355
356

        $fh = fopen($dir . DIRECTORY_SEPARATOR . $filename, 'w');
        fwrite($fh, $this->getSession()->getPage()->getContent());
        fclose($fh);
    }

    /**
     * Determine the full pathname to store a failure-related dump.
     *
     * This is used for content such as the DOM, and screenshots.
     *
Son Nguyen's avatar
Son Nguyen committed
357
     * @param AfterStepScope $scope
Son Nguyen's avatar
Son Nguyen committed
358
359
     * @param String $filetype The file suffix to use. Limited to 4 chars.
     */
Son Nguyen's avatar
Son Nguyen committed
360
    protected function get_faildump_filename(AfterStepScope $scope, $filetype) {
Son Nguyen's avatar
Son Nguyen committed
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
        global $CFG;

        // All the contentdumps should be in the same parent dir.
        if (!$faildumpdir = self::get_run_faildump_dir()) {
            $faildumpdir = self::$faildumpdirname = date('Ymd_His');

            $dir = $CFG->behat_faildump_path . DIRECTORY_SEPARATOR . $faildumpdir;

            if (!is_dir($dir) && !mkdir($dir, $CFG->directorypermissions, true)) {
                // It shouldn't, we already checked that the directory is writable.
                throw new Exception('No directories can be created inside $CFG->behat_faildump_path, check the directory permissions.');
            }
        }
        else {
            // We will always need to know the full path.
            $dir = $CFG->behat_faildump_path . DIRECTORY_SEPARATOR . $faildumpdir;
        }

        // The scenario title + the failed step text.
        // We want a i-am-the-scenario-title_i-am-the-failed-step.$filetype format.
Son Nguyen's avatar
Son Nguyen committed
381
        $filename = $scope->getStep()->getParent()->getTitle() . '_' . $scope->getStep()->getText();
Son Nguyen's avatar
Son Nguyen committed
382
383
384
385
386
387
388
389
390
391
392
393
        $filename = preg_replace('/([^a-zA-Z0-9\_]+)/', '-', $filename);

        // File name limited to 255 characters. Leaving 4 chars for the file
        // extension as we allow .png for images and .html for DOM contents.
        $filename = substr($filename, 0, 250) . '.' . $filetype;

        return array($dir, $filename);
    }

    /**
     * Internal step definition to find exceptions, debugging() messages and PHP debug messages.
     *
Son Nguyen's avatar
Son Nguyen committed
394
     * Part of BehatHooks class as is part of the testing framework, is auto-executed
Son Nguyen's avatar
Son Nguyen committed
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
     * after each step so no features will splicitly use it.
     *
     * @Given /^I look for exceptions$/
     * @throw Exception Unknown type, depending on what we caught in the hook or basic \Exception.
     */
    public function i_look_for_exceptions() {

        // If the step already failed in a hook throw the exception.
        if (!is_null(self::$currentstepexception)) {
            throw self::$currentstepexception;
        }

        // Wrap in try in case we were interacting with a closed window.
        try {

            // Exceptions.
            $exceptionsxpath = "//div[@data-rel='fatalerror']";
            // Debugging messages.
            $debuggingxpath = "//div[@data-rel='debugging']";
            // PHP debug messages.
            $phperrorxpath = "//div[@data-rel='phpdebugmessage']";
            // Any other backtrace.
            $othersxpath = "(//*[contains(., ': call to ')])[1]";

            $xpaths = array($exceptionsxpath, $debuggingxpath, $phperrorxpath, $othersxpath);
            $joinedxpath = implode(' | ', $xpaths);

            // Joined xpath expression. Most of the time there will be no exceptions, so this pre-check
            // is faster than to send the 4 xpath queries for each step.
            if (!$this->getSession()->getDriver()->find($joinedxpath)) {
                return;
            }

            // Exceptions.
            if ($errormsg = $this->getSession()->getPage()->find('xpath', $exceptionsxpath)) {

                // Getting the debugging info and the backtrace.
                $errorinfoboxes = $this->getSession()->getPage()->findAll('css', 'div.alert-error');
                // If errorinfoboxes is empty, try find notifytiny (original) class.
                if (empty($errorinfoboxes)) {
                    $errorinfoboxes = $this->getSession()->getPage()->findAll('css', 'div.notifytiny');
                }
                $errorinfo = $this->get_debug_text($errorinfoboxes[0]->getHtml()) . "\n" .
                    $this->get_debug_text($errorinfoboxes[1]->getHtml());

                $msg = "mahara exception: " . $errormsg->getText() . "\n" . $errorinfo;
                throw new \Exception(html_entity_decode($msg));
            }

            // Debugging messages.
            if ($debuggingmessages = $this->getSession()->getPage()->findAll('xpath', $debuggingxpath)) {
                $msgs = array();
                foreach ($debuggingmessages as $debuggingmessage) {
                    $msgs[] = $this->get_debug_text($debuggingmessage->getHtml());
                }
                $msg = "debugging() message/s found:\n" . implode("\n", $msgs);
                throw new \Exception(html_entity_decode($msg));
            }

            // PHP debug messages.
            if ($phpmessages = $this->getSession()->getPage()->findAll('xpath', $phperrorxpath)) {

                $msgs = array();
                foreach ($phpmessages as $phpmessage) {
                    $msgs[] = $this->get_debug_text($phpmessage->getHtml());
                }
                $msg = "PHP debug message/s found:\n" . implode("\n", $msgs);
                throw new \Exception(html_entity_decode($msg));
            }

            // Any other backtrace.
            // First looking through xpath as it is faster than get and parse the whole page contents,
            // we get the contents and look for matches once we found something to suspect that there is a backtrace.
            if ($this->getSession()->getDriver()->find($othersxpath)) {
                $backtracespattern = '/(line [0-9]* of [^:]*: call to [\->&;:a-zA-Z_\x7f-\xff][\->&;:a-zA-Z0-9_\x7f-\xff]*)/';
                if (preg_match_all($backtracespattern, $this->getSession()->getPage()->getContent(), $backtraces)) {
                    $msgs = array();
                    foreach ($backtraces[0] as $backtrace) {
                        $msgs[] = $backtrace . '()';
                    }
                    $msg = "Other backtraces found:\n" . implode("\n", $msgs);
                    throw new \Exception(htmlentities($msg));
                }
            }

        }
        catch (NoSuchWindow $e) {
            // If we were interacting with a popup window it will not exists after closing it.
        }
    }

    /**
     * Converts HTML tags to line breaks to display the info in CLI
     *
     * @param string $html
     * @return string
     */
    protected function get_debug_text($html) {

        // Replacing HTML tags for new lines and keeping only the text.
        $notags = preg_replace('/<+\s*\/*\s*([A-Z][A-Z0-9]*)\b[^>]*\/*\s*>*/i', "\n", $html);
        return preg_replace("/(\n)+/s", "\n", $notags);
    }

    /**
     * Returns whether the first scenario of the suite is running
     *
     * @return bool
     */
    protected static function is_first_scenario() {
        return !(self::$initprocessesfinished);
    }

    /**
     * Throws an exception after appending an extra info text.
     *
     * @throws Exception
     * @param UnknownError $exception
     * @return void
     */
    protected function throw_unknown_exception(UnknownError $exception) {
        $text = get_string('unknownexceptioninfo', 'tool_behat');
        throw new Exception($text . PHP_EOL . $exception->getMessage());
    }

}