[Summary view] [Print] [Text view]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  /**
  18   * Behat hooks steps definitions.
  19   *
  20   * This methods are used by Behat CLI command.
  21   *
  22   * @package    core
  23   * @category   test
  24   * @copyright  2012 David Monllaó
  25   * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  26   */
  27  
  28  // NOTE: no MOODLE_INTERNAL test here, this file may be required by behat before including /config.php.
  29  
  30  require_once (__DIR__ . '/../../behat/behat_base.php');
  31  
  32  use Behat\Testwork\Hook\Scope\BeforeSuiteScope,
  33      Behat\Testwork\Hook\Scope\AfterSuiteScope,
  34      Behat\Behat\Hook\Scope\BeforeFeatureScope,
  35      Behat\Behat\Hook\Scope\AfterFeatureScope,
  36      Behat\Behat\Hook\Scope\BeforeScenarioScope,
  37      Behat\Behat\Hook\Scope\AfterScenarioScope,
  38      Behat\Behat\Hook\Scope\BeforeStepScope,
  39      Behat\Behat\Hook\Scope\AfterStepScope,
  40      Behat\Mink\Exception\DriverException as DriverException,
  41      WebDriver\Exception\NoSuchWindow as NoSuchWindow,
  42      WebDriver\Exception\UnexpectedAlertOpen as UnexpectedAlertOpen,
  43      WebDriver\Exception\UnknownError as UnknownError,
  44      WebDriver\Exception\CurlExec as CurlExec,
  45      WebDriver\Exception\NoAlertOpenError as NoAlertOpenError;
  46  
  47  /**
  48   * Hooks to the behat process.
  49   *
  50   * Behat accepts hooks after and before each
  51   * suite, feature, scenario and step.
  52   *
  53   * They can not call other steps as part of their process
  54   * like regular steps definitions does.
  55   *
  56   * Throws generic Exception because they are captured by Behat.
  57   *
  58   * @package   core
  59   * @category  test
  60   * @copyright 2012 David Monllaó
  61   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  62   */
  63  class behat_hooks extends behat_base {
  64  
  65      /**
  66       * @var Last browser session start time.
  67       */
  68      protected static $lastbrowsersessionstart = 0;
  69  
  70      /**
  71       * @var For actions that should only run once.
  72       */
  73      protected static $initprocessesfinished = false;
  74  
  75      /**
  76       * Some exceptions can only be caught in a before or after step hook,
  77       * they can not be thrown there as they will provoke a framework level
  78       * failure, but we can store them here to fail the step in i_look_for_exceptions()
  79       * which result will be parsed by the framework as the last step result.
  80       *
  81       * @var Null or the exception last step throw in the before or after hook.
  82       */
  83      protected static $currentstepexception = null;
  84  
  85      /**
  86       * If we are saving any kind of dump on failure we should use the same parent dir during a run.
  87       *
  88       * @var The parent dir name
  89       */
  90      protected static $faildumpdirname = false;
  91  
  92      /**
  93       * Keeps track of time taken by feature to execute.
  94       *
  95       * @var array list of feature timings
  96       */
  97      protected static $timings = array();
  98  
  99      /**
 100       * Hook to capture BeforeSuite event so as to give access to moodle codebase.
 101       * This will try and catch any exception and exists if anything fails.
 102       *
 103       * @param BeforeSuiteScope $scope scope passed by event fired before suite.
 104       * @BeforeSuite
 105       */
 106      public static function before_suite_hook(BeforeSuiteScope $scope) {
 107          try {
 108              self::before_suite($scope);
 109          } catch (behat_stop_exception $e) {
 110              echo $e->getMessage() . PHP_EOL;
 111              exit(1);
 112          }
 113      }
 114  
 115      /**
 116       * Gives access to moodle codebase, ensures all is ready and sets up the test lock.
 117       *
 118       * Includes config.php to use moodle codebase with $CFG->behat_*
 119       * instead of $CFG->prefix and $CFG->dataroot, called once per suite.
 120       *
 121       * @param BeforeSuiteScope $scope scope passed by event fired before suite.
 122       * @static
 123       * @throws behat_stop_exception
 124       */
 125      public static function before_suite(BeforeSuiteScope $scope) {
 126          global $CFG;
 127  
 128          // Defined only when the behat CLI command is running, the moodle init setup process will
 129          // read this value and switch to $CFG->behat_dataroot and $CFG->behat_prefix instead of
 130          // the normal site.
 131          define('BEHAT_TEST', 1);
 132  
 133          define('CLI_SCRIPT', 1);
 134          // With BEHAT_TEST we will be using $CFG->behat_* instead of $CFG->dataroot, $CFG->prefix and $CFG->wwwroot.
 135          require_once(__DIR__ . '/../../../config.php');
 136  
 137          // Now that we are MOODLE_INTERNAL.
 138          require_once (__DIR__ . '/../../behat/classes/behat_command.php');
 139          require_once (__DIR__ . '/../../behat/classes/behat_selectors.php');
 140          require_once (__DIR__ . '/../../behat/classes/behat_context_helper.php');
 141          require_once (__DIR__ . '/../../behat/classes/util.php');
 142          require_once (__DIR__ . '/../../testing/classes/test_lock.php');
 143          require_once (__DIR__ . '/../../testing/classes/nasty_strings.php');
 144  
 145          // Avoids vendor/bin/behat to be executed directly without test environment enabled
 146          // to prevent undesired db & dataroot modifications, this is also checked
 147          // before each scenario (accidental user deletes) in the BeforeScenario hook.
 148  
 149          if (!behat_util::is_test_mode_enabled()) {
 150              throw new behat_stop_exception('Behat only can run if test mode is enabled. More info in ' .
 151                  behat_command::DOCS_URL . '#Running_tests');
 152          }
 153  
 154          // Reset all data, before checking for check_server_status.
 155          // If not done, then it can return apache error, while running tests.
 156          behat_util::reset_all_data();
 157  
 158          // Check if server is running and using same version for cli and apache.
 159          behat_util::check_server_status();
 160  
 161          // Prevents using outdated data, upgrade script would start and tests would fail.
 162          if (!behat_util::is_test_data_updated()) {
 163              $commandpath = 'php admin/tool/behat/cli/init.php';
 164              throw new behat_stop_exception("Your behat test site is outdated, please run\n\n    " .
 165                      $commandpath . "\n\nfrom your moodle dirroot to drop and install the behat test site again.");
 166          }
 167          // Avoid parallel tests execution, it continues when the previous lock is released.
 168          test_lock::acquire('behat');
 169  
 170          // Store the browser reset time if reset after N seconds is specified in config.php.
 171          if (!empty($CFG->behat_restart_browser_after)) {
 172              // Store the initial browser session opening.
 173              self::$lastbrowsersessionstart = time();
 174          }
 175  
 176          if (!empty($CFG->behat_faildump_path) && !is_writable($CFG->behat_faildump_path)) {
 177              throw new behat_stop_exception('You set $CFG->behat_faildump_path to a non-writable directory');
 178          }
 179  
 180          // Handle interrupts on PHP7.
 181          if (extension_loaded('pcntl')) {
 182              $disabled = explode(',', ini_get('disable_functions'));
 183              if (!in_array('pcntl_signal', $disabled)) {
 184                  declare(ticks = 1);
 185              }
 186          }
 187      }
 188  
 189      /**
 190       * Gives access to moodle codebase, to keep track of feature start time.
 191       *
 192       * @param BeforeFeatureScope $scope scope passed by event fired before feature.
 193       * @BeforeFeature
 194       */
 195      public static function before_feature(BeforeFeatureScope $scope) {
 196          if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
 197              return;
 198          }
 199          $file = $scope->getFeature()->getFile();
 200          self::$timings[$file] = microtime(true);
 201      }
 202  
 203      /**
 204       * Gives access to moodle codebase, to keep track of feature end time.
 205       *
 206       * @param AfterFeatureScope $scope scope passed by event fired after feature.
 207       * @AfterFeature
 208       */
 209      public static function after_feature(AfterFeatureScope $scope) {
 210          if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
 211              return;
 212          }
 213          $file = $scope->getFeature()->getFile();
 214          self::$timings[$file] = microtime(true) - self::$timings[$file];
 215          // Probably didn't actually run this, don't output it.
 216          if (self::$timings[$file] < 1) {
 217              unset(self::$timings[$file]);
 218          }
 219      }
 220  
 221      /**
 222       * Gives access to moodle codebase, to keep track of suite timings.
 223       *
 224       * @param AfterSuiteScope $scope scope passed by event fired after suite.
 225       * @AfterSuite
 226       */
 227      public static function after_suite(AfterSuiteScope $scope) {
 228          if (!defined('BEHAT_FEATURE_TIMING_FILE')) {
 229              return;
 230          }
 231          $realroot = realpath(__DIR__.'/../../../').'/';
 232          foreach (self::$timings as $k => $v) {
 233              $new = str_replace($realroot, '', $k);
 234              self::$timings[$new] = round($v, 1);
 235              unset(self::$timings[$k]);
 236          }
 237          if ($existing = @json_decode(file_get_contents(BEHAT_FEATURE_TIMING_FILE), true)) {
 238              self::$timings = array_merge($existing, self::$timings);
 239          }
 240          arsort(self::$timings);
 241          @file_put_contents(BEHAT_FEATURE_TIMING_FILE, json_encode(self::$timings, JSON_PRETTY_PRINT));
 242      }
 243  
 244      /**
 245       * Hook to capture before scenario event to get scope.
 246       *
 247       * @param BeforeScenarioScope $scope scope passed by event fired before scenario.
 248       * @BeforeScenario
 249       */
 250      public function before_scenario_hook(BeforeScenarioScope $scope) {
 251          try {
 252              $this->before_scenario($scope);
 253          } catch (behat_stop_exception $e) {
 254              echo $e->getMessage() . PHP_EOL;
 255              exit(1);
 256          }
 257      }
 258  
 259      /**
 260       * Resets the test environment.
 261       *
 262       * @param BeforeScenarioScope $scope scope passed by event fired before scenario.
 263       * @throws behat_stop_exception If here we are not using the test database it should be because of a coding error
 264       */
 265      public function before_scenario(BeforeScenarioScope $scope) {
 266          global $DB, $SESSION, $CFG;
 267  
 268          // As many checks as we can.
 269          if (!defined('BEHAT_TEST') ||
 270                 !defined('BEHAT_SITE_RUNNING') ||
 271                 php_sapi_name() != 'cli' ||
 272                 !behat_util::is_test_mode_enabled() ||
 273                 !behat_util::is_test_site()) {
 274              throw new behat_stop_exception('Behat only can modify the test database and the test dataroot!');
 275          }
 276  
 277          $moreinfo = 'More info in ' . behat_command::DOCS_URL . '#Running_tests';
 278          $driverexceptionmsg = 'Selenium server is not running, you need to start it to run tests that involve Javascript. ' . $moreinfo;
 279          try {
 280              $session = $this->getSession();
 281          } catch (CurlExec $e) {
 282              // Exception thrown by WebDriver, so only @javascript tests will be caugth; in
 283              // behat_util::check_server_status() we already checked that the server is running.
 284              throw new behat_stop_exception($driverexceptionmsg);
 285          } catch (DriverException $e) {
 286              throw new behat_stop_exception($driverexceptionmsg);
 287          } catch (UnknownError $e) {
 288              // Generic 'I have no idea' Selenium error. Custom exception to provide more feedback about possible solutions.
 289              throw new behat_stop_exception($e->getMessage());
 290          }
 291  
 292          // We need the Mink session to do it and we do it only before the first scenario.
 293          if (self::is_first_scenario()) {
 294              behat_selectors::register_moodle_selectors($session);
 295              behat_context_helper::set_session($scope->getEnvironment());
 296          }
 297  
 298          // Reset mink session between the scenarios.
 299          $session->reset();
 300  
 301          // Reset $SESSION.
 302          \core\session\manager::init_empty_session();
 303  
 304          behat_util::reset_all_data();
 305  
 306          // Assign valid data to admin user (some generator-related code needs a valid user).
 307          $user = $DB->get_record('user', array('username' => 'admin'));
 308          \core\session\manager::set_user($user);
 309  
 310          // Reset the browser if specified in config.php.
 311          if (!empty($CFG->behat_restart_browser_after) && $this->running_javascript()) {
 312              $now = time();
 313              if (self::$lastbrowsersessionstart + $CFG->behat_restart_browser_after < $now) {
 314                  $session->restart();
 315                  self::$lastbrowsersessionstart = $now;
 316              }
 317          }
 318  
 319          // Start always in the the homepage.
 320          try {
 321              // Let's be conservative as we never know when new upstream issues will affect us.
 322              $session->visit($this->locate_path('/'));
 323          } catch (UnknownError $e) {
 324              throw new behat_stop_exception($e->getMessage());
 325          }
 326  
 327  
 328          // Checking that the root path is a Moodle test site.
 329          if (self::is_first_scenario()) {
 330              $notestsiteexception = new behat_stop_exception('The base URL (' . $CFG->wwwroot . ') is not a behat test site, ' .
 331                  'ensure you started the built-in web server in the correct directory or your web server is correctly started and set up');
 332              $this->find("xpath", "//head/child::title[normalize-space(.)='" . behat_util::BEHATSITENAME . "']", $notestsiteexception);
 333  
 334              self::$initprocessesfinished = true;
 335          }
 336          // Run all test with medium (1024x768) screen size, to avoid responsive problems.
 337          $this->resize_window('medium');
 338      }
 339  
 340      /**
 341       * Wait for JS to complete before beginning interacting with the DOM.
 342       *
 343       * Executed only when running against a real browser. We wrap it
 344       * all in a try & catch to forward the exception to i_look_for_exceptions
 345       * so the exception will be at scenario level, which causes a failure, by
 346       * default would be at framework level, which will stop the execution of
 347       * the run.
 348       *
 349       * @param BeforeStepScope $scope scope passed by event fired before step.
 350       * @BeforeStep
 351       */
 352      public function before_step_javascript(BeforeStepScope $scope) {
 353          self::$currentstepexception = null;
 354  
 355          // Only run if JS.
 356          if ($this->running_javascript()) {
 357              try {
 358                  $this->wait_for_pending_js();
 359              } catch (Exception $e) {
 360                  self::$currentstepexception = $e;
 361              }
 362          }
 363      }
 364  
 365      /**
 366       * Wait for JS to complete after finishing the step.
 367       *
 368       * With this we ensure that there are not AJAX calls
 369       * still in progress.
 370       *
 371       * Executed only when running against a real browser. We wrap it
 372       * all in a try & catch to forward the exception to i_look_for_exceptions
 373       * so the exception will be at scenario level, which causes a failure, by
 374       * default would be at framework level, which will stop the execution of
 375       * the run.
 376       *
 377       * @param AfterStepScope $scope scope passed by event fired after step..
 378       * @AfterStep
 379       */
 380      public function after_step_javascript(AfterStepScope $scope) {
 381          global $CFG, $DB;
 382  
 383          // Save the page content if the step failed.
 384          if (!empty($CFG->behat_faildump_path) &&
 385              $scope->getTestResult()->getResultCode() === Behat\Testwork\Tester\Result\TestResult::FAILED) {
 386              $this->take_contentdump($scope);
 387          }
 388  
 389          // Abort any open transactions to prevent subsequent tests hanging.
 390          // This does the same as abort_all_db_transactions(), but doesn't call error_log() as we don't
 391          // want to see a message in the behat output.
 392          if (($scope->getTestResult() instanceof \Behat\Behat\Tester\Result\ExecutedStepResult) &&
 393              $scope->getTestResult()->hasException()) {
 394              if ($DB && $DB->is_transaction_started()) {
 395                  $DB->force_transaction_rollback();
 396              }
 397          }
 398  
 399          // Only run if JS.
 400          if (!$this->running_javascript()) {
 401              return;
 402          }
 403  
 404          // Save a screenshot if the step failed.
 405          if (!empty($CFG->behat_faildump_path) &&
 406              $scope->getTestResult()->getResultCode() === Behat\Testwork\Tester\Result\TestResult::FAILED) {
 407              $this->take_screenshot($scope);
 408          }
 409  
 410          try {
 411              $this->wait_for_pending_js();
 412              self::$currentstepexception = null;
 413          } catch (UnexpectedAlertOpen $e) {
 414              self::$currentstepexception = $e;
 415  
 416              // Accepting the alert so the framework can continue properly running
 417              // the following scenarios. Some browsers already closes the alert, so
 418              // wrapping in a try & catch.
 419              try {
 420                  $this->getSession()->getDriver()->getWebDriverSession()->accept_alert();
 421              } catch (Exception $e) {
 422                  // Catching the generic one as we never know how drivers reacts here.
 423              }
 424          } catch (Exception $e) {
 425              self::$currentstepexception = $e;
 426          }
 427      }
 428  
 429      /**
 430       * Executed after scenario having switch window to restart session.
 431       * This is needed to close all extra browser windows and starting
 432       * one browser window.
 433       *
 434       * @param AfterScenarioScope $scope scope passed by event fired after scenario.
 435       * @AfterScenario @_switch_window
 436       */
 437      public function after_scenario_switchwindow(AfterScenarioScope $scope) {
 438          for ($count = 0; $count < self::EXTENDED_TIMEOUT; $count++) {
 439              try {
 440                  $this->getSession()->restart();
 441                  break;
 442              } catch (DriverException $e) {
 443                  // Wait for timeout and try again.
 444                  sleep(self::TIMEOUT);
 445              }
 446          }
 447          // If session is not restarted above then it will try to start session before next scenario
 448          // and if that fails then exception will be thrown.
 449      }
 450  
 451      /**
 452       * Getter for self::$faildumpdirname
 453       *
 454       * @return string
 455       */
 456      protected function get_run_faildump_dir() {
 457          return self::$faildumpdirname;
 458      }
 459  
 460      /**
 461       * Take screenshot when a step fails.
 462       *
 463       * @throws Exception
 464       * @param AfterStepScope $scope scope passed by event after step.
 465       */
 466      protected function take_screenshot(AfterStepScope $scope) {
 467          // Goutte can't save screenshots.
 468          if (!$this->running_javascript()) {
 469              return false;
 470          }
 471  
 472          // Some drivers (e.g. chromedriver) may throw an exception while trying to take a screenshot.  If this isn't handled,
 473          // the behat run dies.  We don't want to lose the information about the failure that triggered the screenshot,
 474          // so let's log the exception message to a file (to explain why there's no screenshot) and allow the run to continue,
 475          // handling the failure as normal.
 476          try {
 477              list ($dir, $filename) = $this->get_faildump_filename($scope, 'png');
 478              $this->saveScreenshot($filename, $dir);
 479          } catch (Exception $e) {
 480              // Catching all exceptions as we don't know what the driver might throw.
 481              list ($dir, $filename) = $this->get_faildump_filename($scope, 'txt');
 482              $message = "Could not save screenshot due to an error\n" . $e->getMessage();
 483              file_put_contents($dir . DIRECTORY_SEPARATOR . $filename, $message);
 484          }
 485      }
 486  
 487      /**
 488       * Take a dump of the page content when a step fails.
 489       *
 490       * @throws Exception
 491       * @param AfterStepScope $scope scope passed by event after step.
 492       */
 493      protected function take_contentdump(AfterStepScope $scope) {
 494          list ($dir, $filename) = $this->get_faildump_filename($scope, 'html');
 495  
 496          try {
 497              // Driver may throw an exception during getContent(), so do it first to avoid getting an empty file.
 498              $content = $this->getSession()->getPage()->getContent();
 499          } catch (Exception $e) {
 500              // Catching all exceptions as we don't know what the driver might throw.
 501              $content = "Could not save contentdump due to an error\n" . $e->getMessage();
 502          }
 503          file_put_contents($dir . DIRECTORY_SEPARATOR . $filename, $content);
 504      }
 505  
 506      /**
 507       * Determine the full pathname to store a failure-related dump.
 508       *
 509       * This is used for content such as the DOM, and screenshots.
 510       *
 511       * @param AfterStepScope $scope scope passed by event after step.
 512       * @param String $filetype The file suffix to use. Limited to 4 chars.
 513       */
 514      protected function get_faildump_filename(AfterStepScope $scope, $filetype) {
 515          global $CFG;
 516  
 517          // All the contentdumps should be in the same parent dir.
 518          if (!$faildumpdir = self::get_run_faildump_dir()) {
 519              $faildumpdir = self::$faildumpdirname = date('Ymd_His');
 520  
 521              $dir = $CFG->behat_faildump_path . DIRECTORY_SEPARATOR . $faildumpdir;
 522  
 523              if (!is_dir($dir) && !mkdir($dir, $CFG->directorypermissions, true)) {
 524                  // It shouldn't, we already checked that the directory is writable.
 525                  throw new Exception('No directories can be created inside $CFG->behat_faildump_path, check the directory permissions.');
 526              }
 527          } else {
 528              // We will always need to know the full path.
 529              $dir = $CFG->behat_faildump_path . DIRECTORY_SEPARATOR . $faildumpdir;
 530          }
 531  
 532          // The scenario title + the failed step text.
 533          // We want a i-am-the-scenario-title_i-am-the-failed-step.$filetype format.
 534          $filename = $scope->getFeature()->getTitle() . '_' . $scope->getStep()->getText();
 535          $filename = preg_replace('/([^a-zA-Z0-9\_]+)/', '-', $filename);
 536  
 537          // File name limited to 255 characters. Leaving 5 chars for line number and 4 chars for the file.
 538          // extension as we allow .png for images and .html for DOM contents.
 539          $filename = substr($filename, 0, 245) . '_' . $scope->getStep()->getLine() . '.' . $filetype;
 540  
 541          return array($dir, $filename);
 542      }
 543  
 544      /**
 545       * Internal step definition to find exceptions, debugging() messages and PHP debug messages.
 546       *
 547       * Part of behat_hooks class as is part of the testing framework, is auto-executed
 548       * after each step so no features will splicitly use it.
 549       *
 550       * @Given /^I look for exceptions$/
 551       * @throw Exception Unknown type, depending on what we caught in the hook or basic \Exception.
 552       * @see Moodle\BehatExtension\Tester\MoodleStepTester
 553       */
 554      public function i_look_for_exceptions() {
 555          // If the step already failed in a hook throw the exception.
 556          if (!is_null(self::$currentstepexception)) {
 557              throw self::$currentstepexception;
 558          }
 559  
 560          $this->look_for_exceptions();
 561      }
 562  
 563      /**
 564       * Returns whether the first scenario of the suite is running
 565       *
 566       * @return bool
 567       */
 568      protected static function is_first_scenario() {
 569          return !(self::$initprocessesfinished);
 570      }
 571  }
 572  
 573  /**
 574   * Behat stop exception
 575   *
 576   * This exception is thrown from before suite or scenario if any setup problem found.
 577   *
 578   * @package    core_test
 579   * @copyright  2016 Rajesh Taneja <rajesh@moodle.com>
 580   * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
 581   */
 582  class behat_stop_exception extends \Exception{}