[Summary view] [Print] [Text view]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  /**
  18   * Wrapper to run previously set-up behat tests in parallel.
  19   *
  20   * @package    tool_behat
  21   * @copyright  2014 NetSpot Pty Ltd
  22   * @license    http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  23   */
  24  
  25  if (isset($_SERVER['REMOTE_ADDR'])) {
  26      die(); // No access from web!
  27  }
  28  
  29  define('BEHAT_UTIL', true);
  30  define('CLI_SCRIPT', true);
  31  define('ABORT_AFTER_CONFIG', true);
  32  define('CACHE_DISABLE_ALL', true);
  33  define('NO_OUTPUT_BUFFERING', true);
  34  
  35  require_once(__DIR__ .'/../../../../config.php');
  36  require_once (__DIR__.'/../../../../lib/clilib.php');
  37  require_once (__DIR__.'/../../../../lib/behat/lib.php');
  38  require_once (__DIR__.'/../../../../lib/behat/classes/behat_command.php');
  39  require_once (__DIR__.'/../../../../lib/behat/classes/behat_config_manager.php');
  40  
  41  error_reporting(E_ALL | E_STRICT);
  42  ini_set('display_errors', '1');
  43  ini_set('log_errors', '1');
  44  
  45  list($options, $unrecognised) = cli_get_params(
  46      array(
  47          'stop-on-failure' => 0,
  48          'verbose'  => false,
  49          'replace'  => false,
  50          'help'     => false,
  51          'tags'     => '',
  52          'profile'  => '',
  53          'feature'  => '',
  54          'fromrun'  => 1,
  55          'torun'    => 0,
  56          'single-run' => false,
  57      ),
  58      array(
  59          'h' => 'help',
  60          't' => 'tags',
  61          'p' => 'profile',
  62          's' => 'single-run',
  63      )
  64  );
  65  
  66  // Checking run.php CLI script usage.
  67  $help = "
  68  Behat utilities to run behat tests in parallel
  69  
  70  Usage:
  71    php run.php [--BEHAT_OPTION=\"value\"] [--feature=\"value\"] [--replace] [--fromrun=value --torun=value] [--help]
  72  
  73  Options:
  74  --BEHAT_OPTION     Any combination of behat option specified in http://behat.readthedocs.org/en/v2.5/guides/6.cli.html
  75  --feature          Only execute specified feature file (Absolute path of feature file).
  76  --replace          Replace args string with run process number, useful for output.
  77  --fromrun          Execute run starting from (Used for parallel runs on different vms)
  78  --torun            Execute run till (Used for parallel runs on different vms)
  79  
  80  -h, --help         Print out this help
  81  
  82  Example from Moodle root directory:
  83  \$ php admin/tool/behat/cli/run.php --tags=\"@javascript\"
  84  
  85  More info in http://docs.moodle.org/dev/Acceptance_testing#Running_tests
  86  ";
  87  
  88  if (!empty($options['help'])) {
  89      echo $help;
  90      exit(0);
  91  }
  92  
  93  $parallelrun = behat_config_manager::get_parallel_test_runs($options['fromrun']);
  94  
  95  // Default torun is maximum parallel runs.
  96  if (empty($options['torun'])) {
  97      $options['torun'] = $parallelrun;
  98  }
  99  
 100  // Capture signals and ensure we clean symlinks.
 101  if (extension_loaded('pcntl')) {
 102      $disabled = explode(',', ini_get('disable_functions'));
 103      if (!in_array('pcntl_signal', $disabled)) {
 104          // Handle interrupts on PHP7.
 105          declare(ticks = 1);
 106  
 107          pcntl_signal(SIGTERM, "signal_handler");
 108          pcntl_signal(SIGINT, "signal_handler");
 109      }
 110  }
 111  
 112  $time = microtime(true);
 113  array_walk($unrecognised, function (&$v) {
 114      if ($x = preg_filter("#^(-+\w+)=(.+)#", "\$1=\"\$2\"", $v)) {
 115          $v = $x;
 116      } else if (!preg_match("#^-#", $v)) {
 117          $v = escapeshellarg($v);
 118      }
 119  });
 120  $extraopts = $unrecognised;
 121  
 122  $tags = '';
 123  
 124  if ($options['profile']) {
 125      $profile = $options['profile'];
 126  
 127      // If profile passed is not set, then exit.
 128      if (!isset($CFG->behat_config[$profile]) && !isset($CFG->behat_profiles[$profile]) &&
 129          !(isset($options['replace']) && (strpos($options['profile'], $options['replace']) >= 0 ))) {
 130          echo "Invalid profile passed: " . $profile . PHP_EOL;
 131          exit(1);
 132      }
 133  
 134      $extraopts[] = '--profile="' . $profile . '"';
 135      // By default, profile tags will be used.
 136      if (!empty($CFG->behat_config[$profile]['filters']['tags'])) {
 137          $tags = $CFG->behat_config[$profile]['filters']['tags'];
 138      }
 139  }
 140  
 141  // Command line tags have precedence (std behat behavior).
 142  if ($options['tags']) {
 143      $tags = $options['tags'];
 144      $extraopts[] = '--tags="' . $tags . '"';
 145  }
 146  
 147  // Feature should be added to last, for behat command.
 148  if ($options['feature']) {
 149      $extraopts[] = $options['feature'];
 150      // Only run 1 process as process.
 151      // Feature file is picked from absolute path provided, so no need to check for behat.yml.
 152      $options['torun'] = $options['fromrun'];
 153  }
 154  
 155  // Set of options to pass to behat.
 156  $extraopts = implode(' ', $extraopts);
 157  
 158  // If empty parallelrun then just check with user if it's a run single behat test.
 159  if (empty($parallelrun)) {
 160      $cwd = getcwd();
 161      chdir(__DIR__);
 162      $runtestscommand = behat_command::get_behat_command(false, false, true);
 163      $runtestscommand .= ' --config ' . behat_config_manager::get_behat_cli_config_filepath();
 164      $runtestscommand .= ' ' . $extraopts;
 165      echo "Running single behat site:" . PHP_EOL;
 166      passthru("php $runtestscommand", $code);
 167      chdir($cwd);
 168      exit($code);
 169  }
 170  
 171  // Update config file if tags defined.
 172  if ($tags) {
 173      // Hack to set proper dataroot and wwwroot.
 174      $behatdataroot = $CFG->behat_dataroot;
 175      $behatwwwroot  = $CFG->behat_wwwroot;
 176      for ($i = 1; $i <= $parallelrun; $i++) {
 177          $CFG->behatrunprocess = $i;
 178  
 179          if (!empty($CFG->behat_parallel_run[$i - 1]['behat_wwwroot'])) {
 180              $CFG->behat_wwwroot = $CFG->behat_parallel_run[$i - 1]['behat_wwwroot'];
 181          } else {
 182              $CFG->behat_wwwroot = $behatwwwroot . "/" . BEHAT_PARALLEL_SITE_NAME . $i;
 183          }
 184          if (!empty($CFG->behat_parallel_run[$i - 1]['behat_dataroot'])) {
 185              $CFG->behat_dataroot = $CFG->behat_parallel_run[$i - 1]['behat_dataroot'];
 186          } else {
 187              $CFG->behat_dataroot = $behatdataroot . $i;
 188          }
 189          behat_config_manager::update_config_file('', true, $tags);
 190      }
 191      $CFG->behat_dataroot = $behatdataroot;
 192      $CFG->behat_wwwroot = $behatwwwroot;
 193      unset($CFG->behatrunprocess);
 194  }
 195  
 196  $cmds = array();
 197  echo "Running " . ($options['torun'] - $options['fromrun'] + 1) . " parallel behat sites:" . PHP_EOL;
 198  
 199  for ($i = $options['fromrun']; $i <= $options['torun']; $i++) {
 200      $CFG->behatrunprocess = $i;
 201  
 202      // Options parameters to be added to each run.
 203      $myopts = !empty($options['replace']) ? str_replace($options['replace'], $i, $extraopts) : $extraopts;
 204  
 205      $behatcommand = behat_command::get_behat_command(false, false, true);
 206      $behatconfigpath = behat_config_manager::get_behat_cli_config_filepath($i);
 207  
 208      // Command to execute behat run.
 209      $cmds[BEHAT_PARALLEL_SITE_NAME . $i] = $behatcommand . ' --config ' . $behatconfigpath . " " . $myopts;
 210      echo "[" . BEHAT_PARALLEL_SITE_NAME . $i . "] " . $cmds[BEHAT_PARALLEL_SITE_NAME . $i] . PHP_EOL;
 211  }
 212  
 213  if (empty($cmds)) {
 214      echo "No commands to execute " . PHP_EOL;
 215      exit(1);
 216  }
 217  
 218  // Create site symlink if necessary.
 219  if (!behat_config_manager::create_parallel_site_links($options['fromrun'], $options['torun'])) {
 220      echo "Check permissions. If on windows, make sure you are running this command as admin" . PHP_EOL;
 221      exit(1);
 222  }
 223  
 224  // Execute all commands.
 225  $processes = cli_execute_parallel($cmds, __DIR__);
 226  $stoponfail = empty($options['stop-on-failure']) ? false : true;
 227  
 228  // Print header.
 229  print_process_start_info($processes);
 230  
 231  // Print combined run o/p from processes.
 232  $exitcodes = print_combined_run_output($processes, $stoponfail);
 233  $time = round(microtime(true) - $time, 1);
 234  echo "Finished in " . gmdate("G\h i\m s\s", $time) . PHP_EOL . PHP_EOL;
 235  
 236  ksort($exitcodes);
 237  
 238  // Print exit info from each run.
 239  // Status bits contains pass/fail status of parallel runs.
 240  $status = 0;
 241  $processcounter = 0;
 242  foreach ($exitcodes as $exitcode) {
 243      if ($exitcode) {
 244          $status |= (1 << $processcounter);
 245      }
 246      $processcounter++;
 247  }
 248  
 249  // Run finished. Show exit code and output from individual process.
 250  $verbose = empty($options['verbose']) ? false : true;
 251  $verbose = $verbose || !empty($status);
 252  
 253  // Show exit code from each process, if any process failed.
 254  if ($verbose) {
 255      // Echo exit codes.
 256      echo "Exit codes for each behat run: " . PHP_EOL;
 257      foreach ($exitcodes as $run => $exitcode) {
 258          echo $run . ": " . $exitcode . PHP_EOL;
 259      }
 260  
 261      // Show failed re-run commands.
 262      if ($status) {
 263          echo "To re-run failed processes, you can use following commands:" . PHP_EOL;
 264          foreach ($cmds as $name => $cmd) {
 265              if (!empty($exitcodes[$name])) {
 266                  echo "[" . $name . "] " . $cmd . PHP_EOL;
 267              }
 268          }
 269      }
 270      echo PHP_EOL;
 271  }
 272  
 273  print_each_process_info($processes, $verbose);
 274  
 275  // Remove site symlink if necessary.
 276  behat_config_manager::drop_parallel_site_links();
 277  
 278  exit($status);
 279  
 280  /**
 281   * Signal handler for terminal exit.
 282   *
 283   * @param int $signal signal number.
 284   */
 285  function signal_handler($signal) {
 286      switch ($signal) {
 287          case SIGTERM:
 288          case SIGKILL:
 289          case SIGINT:
 290              // Remove site symlink if necessary.
 291              behat_config_manager::drop_parallel_site_links();
 292              exit(1);
 293      }
 294  }
 295  
 296  /**
 297   * Prints header from the first process.
 298   *
 299   * @param array $processes list of processes to loop though.
 300   */
 301  function print_process_start_info($processes) {
 302      $printed = false;
 303      // Keep looping though processes, till we get first process o/p.
 304      while (!$printed) {
 305          usleep(10000);
 306          foreach ($processes as $name => $process) {
 307              // Exit if any process has stopped.
 308              if (!$process->isRunning()) {
 309                  $printed = true;
 310                  break;
 311              }
 312  
 313              $op = explode(PHP_EOL, $process->getOutput());
 314              if (count($op) >= 3) {
 315                  foreach ($op as $line) {
 316                      if (trim($line) && (strpos($line, '.') !== 0)) {
 317                          echo $line . PHP_EOL;
 318                      }
 319                  }
 320                  $printed = true;
 321              }
 322          }
 323      }
 324  }
 325  
 326  /**
 327   * Loop though all processes and print combined o/p
 328   *
 329   * @param array $processes list of processes to loop though.
 330   * @param bool $stoponfail Stop all processes and exit if failed.
 331   * @return array list of exit codes from all processes.
 332   */
 333  function print_combined_run_output($processes, $stoponfail = false) {
 334      $exitcodes = array();
 335      $maxdotsonline = 70;
 336      $remainingprintlen = $maxdotsonline;
 337      $progresscount = 0;
 338      while (count($exitcodes) != count($processes)) {
 339          usleep(10000);
 340          foreach ($processes as $name => $process) {
 341              if ($process->isRunning()) {
 342                  $op = $process->getIncrementalOutput();
 343                  if (trim($op)) {
 344                      $update = preg_filter('#^\s*([FS\.\-]+)(?:\s+\d+)?\s*$#', '$1', $op);
 345                      // Exit process if anything fails.
 346                      if ($stoponfail && (strpos($update, 'F') !== false)) {
 347                          $process->stop(0);
 348                      }
 349  
 350                      $strlentoprint = strlen($update);
 351  
 352                      // If not enough dots printed on line then just print.
 353                      if ($strlentoprint < $remainingprintlen) {
 354                          echo $update;
 355                          $remainingprintlen = $remainingprintlen - $strlentoprint;
 356                      } else if ($strlentoprint == $remainingprintlen) {
 357                          $progresscount += $maxdotsonline;
 358                          echo $update ." " . $progresscount . PHP_EOL;
 359                          $remainingprintlen = $maxdotsonline;
 360                      } else {
 361                          while ($part = substr($update, 0, $remainingprintlen) > 0) {
 362                              $progresscount += $maxdotsonline;
 363                              echo $part . " " . $progresscount . PHP_EOL;
 364                              $update = substr($update, $remainingprintlen);
 365                              $remainingprintlen = $maxdotsonline;
 366                          }
 367                      }
 368                  }
 369              } else {
 370                  $exitcodes[$name] = $process->getExitCode();
 371                  if ($stoponfail && ($exitcodes[$name] != 0)) {
 372                      foreach ($processes as $l => $p) {
 373                          $exitcodes[$l] = -1;
 374                          $process->stop(0);
 375                      }
 376                  }
 377              }
 378          }
 379      }
 380  
 381      echo PHP_EOL;
 382      return $exitcodes;
 383  }
 384  
 385  /**
 386   * Loop though all processes and print combined o/p
 387   *
 388   * @param array $processes list of processes to loop though.
 389   * @param bool $verbose Show verbose output for each process.
 390   */
 391  function print_each_process_info($processes, $verbose = false) {
 392      foreach ($processes as $name => $process) {
 393          echo "**************** [" . $name . "] ****************" . PHP_EOL;
 394          if ($verbose) {
 395              echo $process->getOutput();
 396              echo $process->getErrorOutput();
 397          } else {
 398              $op = explode(PHP_EOL, $process->getOutput());
 399              foreach ($op as $line) {
 400                  // Don't print progress .
 401                  if (trim($line) && (strpos($line, '.') !== 0) && (strpos($line, 'Moodle ') !== 0) &&
 402                      (strpos($line, 'Server OS ') !== 0) && (strpos($line, 'Started at ') !== 0) &&
 403                      (strpos($line, 'Browser specific fixes ') !== 0)) {
 404                      echo $line . PHP_EOL;
 405                  }
 406              }
 407          }
 408          echo PHP_EOL;
 409      }
 410  }