[Summary view] [Print] [Text view]

   1  <?php
   2  // This file is part of Moodle - http://moodle.org/
   3  //
   4  // Moodle is free software: you can redistribute it and/or modify
   5  // it under the terms of the GNU General Public License as published by
   6  // the Free Software Foundation, either version 3 of the License, or
   7  // (at your option) any later version.
   8  //
   9  // Moodle is distributed in the hope that it will be useful,
  10  // but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  // GNU General Public License for more details.
  13  //
  14  // You should have received a copy of the GNU General Public License
  15  // along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
  16  
  17  /**
  18   * A class for loading and preparing grade data from import.
  19   *
  20   * @package   gradeimport_csv
  21   * @copyright 2014 Adrian Greeve <adrian@moodle.com>
  22   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  23   */
  24  
  25  defined('MOODLE_INTERNAL') || die();
  26  
  27  /**
  28   * A class for loading and preparing grade data from import.
  29   *
  30   * @package   gradeimport_csv
  31   * @copyright 2014 Adrian Greeve <adrian@moodle.com>
  32   * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
  33   */
  34  class gradeimport_csv_load_data {
  35  
  36      /** @var string $error csv import error. */
  37      protected $error;
  38      /** @var int $iid Unique identifier for these csv records. */
  39      protected $iid;
  40      /** @var array $headers Column names for the data. */
  41      protected $headers;
  42      /** @var array $previewdata A subsection of the csv imported data. */
  43      protected $previewdata;
  44  
  45      // The map_user_data_with_value variables.
  46      /** @var array $newgrades Grades to be inserted into the gradebook. */
  47      protected $newgrades;
  48      /** @var array $newfeedbacks Feedback to be inserted into the gradebook. */
  49      protected $newfeedbacks;
  50      /** @var int $studentid Student ID*/
  51      protected $studentid;
  52  
  53      // The prepare_import_grade_data() variables.
  54      /** @var bool $status The current status of the import. True = okay, False = errors. */
  55      protected $status;
  56      /** @var int $importcode The code for this batch insert. */
  57      protected $importcode;
  58      /** @var array $gradebookerrors An array of errors from trying to import into the gradebook. */
  59      protected $gradebookerrors;
  60      /** @var array $newgradeitems An array of new grade items to be inserted into the gradebook. */
  61      protected $newgradeitems;
  62  
  63      /**
  64       * Load CSV content for previewing.
  65       *
  66       * @param string $text The grade data being imported.
  67       * @param string $encoding The type of encoding the file uses.
  68       * @param string $separator The separator being used to define each field.
  69       * @param int $previewrows How many rows are being previewed.
  70       */
  71      public function load_csv_content($text, $encoding, $separator, $previewrows) {
  72          $this->raise_limits();
  73  
  74          $this->iid = csv_import_reader::get_new_iid('grade');
  75          $csvimport = new csv_import_reader($this->iid, 'grade');
  76  
  77          $csvimport->load_csv_content($text, $encoding, $separator);
  78          $this->error = $csvimport->get_error();
  79  
  80          // If there are no import errors then proceed.
  81          if (empty($this->error)) {
  82  
  83              // Get header (field names).
  84              $this->headers = $csvimport->get_columns();
  85              $this->trim_headers();
  86  
  87              $csvimport->init();
  88              $this->previewdata = array();
  89  
  90              for ($numlines = 0; $numlines <= $previewrows; $numlines++) {
  91                  $lines = $csvimport->next();
  92                  if ($lines) {
  93                      $this->previewdata[] = $lines;
  94                  }
  95              }
  96          }
  97      }
  98  
  99      /**
 100       * Gets all of the grade items in this course.
 101       *
 102       * @param int $courseid Course id;
 103       * @return array An array of grade items for the course.
 104       */
 105      public static function fetch_grade_items($courseid) {
 106          $gradeitems = null;
 107          if ($allgradeitems = grade_item::fetch_all(array('courseid' => $courseid))) {
 108              foreach ($allgradeitems as $gradeitem) {
 109                  // Skip course type and category type.
 110                  if ($gradeitem->itemtype == 'course' || $gradeitem->itemtype == 'category') {
 111                      continue;
 112                  }
 113  
 114                  $displaystring = null;
 115                  if (!empty($gradeitem->itemmodule)) {
 116                      $displaystring = get_string('modulename', $gradeitem->itemmodule).get_string('labelsep', 'langconfig')
 117                              .$gradeitem->get_name();
 118                  } else {
 119                      $displaystring = $gradeitem->get_name();
 120                  }
 121                  $gradeitems[$gradeitem->id] = $displaystring;
 122              }
 123          }
 124          return $gradeitems;
 125      }
 126  
 127      /**
 128       * Cleans the column headers from the CSV file.
 129       */
 130      protected function trim_headers() {
 131          foreach ($this->headers as $i => $h) {
 132              $h = trim($h); // Remove whitespace.
 133              $h = clean_param($h, PARAM_RAW); // Clean the header.
 134              $this->headers[$i] = $h;
 135          }
 136      }
 137  
 138      /**
 139       * Raises the php execution time and memory limits for importing the CSV file.
 140       */
 141      protected function raise_limits() {
 142          // Large files are likely to take their time and memory. Let PHP know
 143          // that we'll take longer, and that the process should be recycled soon
 144          // to free up memory.
 145          core_php_time_limit::raise();
 146          raise_memory_limit(MEMORY_EXTRA);
 147      }
 148  
 149      /**
 150       * Inserts a record into the grade_import_values table. This also adds common record information.
 151       *
 152       * @param object $record The grade record being inserted into the database.
 153       * @param int $studentid The student ID.
 154       * @return bool|int true or insert id on success. Null if the grade value is too high.
 155       */
 156      protected function insert_grade_record($record, $studentid) {
 157          global $DB, $USER, $CFG;
 158          $record->importcode = $this->importcode;
 159          $record->userid     = $studentid;
 160          $record->importer   = $USER->id;
 161          // By default the maximum grade is 100.
 162          $gradepointmaximum = 100;
 163          // If the grade limit has been increased then use the gradepointmax setting.
 164          if ($CFG->unlimitedgrades) {
 165              $gradepointmaximum = $CFG->gradepointmax;
 166          }
 167          // If the record final grade is set then check that the grade value isn't too high.
 168          // Final grade will not be set if we are inserting feedback.
 169          if (!isset($record->finalgrade) || $record->finalgrade <= $gradepointmaximum) {
 170              return $DB->insert_record('grade_import_values', $record);
 171          } else {
 172              $this->cleanup_import(get_string('gradevaluetoobig', 'grades', $gradepointmaximum));
 173              return null;
 174          }
 175      }
 176  
 177      /**
 178       * Insert the new grade into the grade item buffer table.
 179       *
 180       * @param array $header The column headers from the CSV file.
 181       * @param int $key Current row identifier.
 182       * @param string $value The value for this row (final grade).
 183       * @return stdClass new grade that is ready for commiting to the gradebook.
 184       */
 185      protected function import_new_grade_item($header, $key, $value) {
 186          global $DB, $USER;
 187  
 188          // First check if header is already in temp database.
 189          if (empty($this->newgradeitems[$key])) {
 190  
 191              $newgradeitem = new stdClass();
 192              $newgradeitem->itemname = $header[$key];
 193              $newgradeitem->importcode = $this->importcode;
 194              $newgradeitem->importer = $USER->id;
 195  
 196              // Insert into new grade item buffer.
 197              $this->newgradeitems[$key] = $DB->insert_record('grade_import_newitem', $newgradeitem);
 198          }
 199          $newgrade = new stdClass();
 200          $newgrade->newgradeitem = $this->newgradeitems[$key];
 201  
 202          $trimmed = trim($value);
 203          if ($trimmed === '' or $trimmed == '-') {
 204              // Blank or dash grade means null, ie "no grade".
 205              $newgrade->finalgrade = null;
 206          } else {
 207              // We have an actual grade.
 208              $newgrade->finalgrade = $value;
 209          }
 210          $this->newgrades[] = $newgrade;
 211          return $newgrade;
 212      }
 213  
 214      /**
 215       * Check that the user is in the system.
 216       *
 217       * @param string $value The value, from the csv file, being mapped to identify the user.
 218       * @param array $userfields Contains the field and label being mapped from.
 219       * @return int Returns the user ID if it exists, otherwise null.
 220       */
 221      protected function check_user_exists($value, $userfields) {
 222          global $DB;
 223  
 224          $usercheckproblem = false;
 225          $user = null;
 226          // The user may use the incorrect field to match the user. This could result in an exception.
 227          try {
 228              $user = $DB->get_record('user', array($userfields['field'] => $value));
 229          } catch (Exception $e) {
 230              $usercheckproblem = true;
 231          }
 232          // Field may be fine, but no records were returned.
 233          if (!$user || $usercheckproblem) {
 234              $usermappingerrorobj = new stdClass();
 235              $usermappingerrorobj->field = $userfields['label'];
 236              $usermappingerrorobj->value = $value;
 237              $this->cleanup_import(get_string('usermappingerror', 'grades', $usermappingerrorobj));
 238              unset($usermappingerrorobj);
 239              return null;
 240          }
 241          return $user->id;
 242      }
 243  
 244      /**
 245       * Check to see if the feedback matches a grade item.
 246       *
 247       * @param int $courseid The course ID.
 248       * @param int $itemid The ID of the grade item that the feedback relates to.
 249       * @param string $value The actual feedback being imported.
 250       * @return object Creates a feedback object with the item ID and the feedback value.
 251       */
 252      protected function create_feedback($courseid, $itemid, $value) {
 253          // Case of an id, only maps id of a grade_item.
 254          // This was idnumber.
 255          if (!new grade_item(array('id' => $itemid, 'courseid' => $courseid))) {
 256              // Supplied bad mapping, should not be possible since user
 257              // had to pick mapping.
 258              $this->cleanup_import(get_string('importfailed', 'grades'));
 259              return null;
 260          }
 261  
 262          // The itemid is the id of the grade item.
 263          $feedback = new stdClass();
 264          $feedback->itemid   = $itemid;
 265          $feedback->feedback = $value;
 266          return $feedback;
 267      }
 268  
 269      /**
 270       * This updates existing grade items.
 271       *
 272       * @param int $courseid The course ID.
 273       * @param array $map Mapping information provided by the user.
 274       * @param int $key The line that we are currently working on.
 275       * @param bool $verbosescales Form setting for grading with scales.
 276       * @param string $value The grade value.
 277       * @return array grades to be updated.
 278       */
 279      protected function update_grade_item($courseid, $map, $key, $verbosescales, $value) {
 280          // Case of an id, only maps id of a grade_item.
 281          // This was idnumber.
 282          if (!$gradeitem = new grade_item(array('id' => $map[$key], 'courseid' => $courseid))) {
 283              // Supplied bad mapping, should not be possible since user
 284              // had to pick mapping.
 285              $this->cleanup_import(get_string('importfailed', 'grades'));
 286              return null;
 287          }
 288  
 289          // Check if grade item is locked if so, abort.
 290          if ($gradeitem->is_locked()) {
 291              $this->cleanup_import(get_string('gradeitemlocked', 'grades'));
 292              return null;
 293          }
 294  
 295          $newgrade = new stdClass();
 296          $newgrade->itemid = $gradeitem->id;
 297          if ($gradeitem->gradetype == GRADE_TYPE_SCALE and $verbosescales) {
 298              if ($value === '' or $value == '-') {
 299                  $value = null; // No grade.
 300              } else {
 301                  $scale = $gradeitem->load_scale();
 302                  $scales = explode(',', $scale->scale);
 303                  $scales = array_map('trim', $scales); // Hack - trim whitespace around scale options.
 304                  array_unshift($scales, '-'); // Scales start at key 1.
 305                  $key = array_search($value, $scales);
 306                  if ($key === false) {
 307                      $this->cleanup_import(get_string('badgrade', 'grades'));
 308                      return null;
 309                  }
 310                  $value = $key;
 311              }
 312              $newgrade->finalgrade = $value;
 313          } else {
 314              if ($value === '' or $value == '-') {
 315                  $value = null; // No grade.
 316              } else {
 317                  // If the value has a local decimal or can correctly be unformatted, do it.
 318                  $validvalue = unformat_float($value, true);
 319                  if ($validvalue !== false) {
 320                      $value = $validvalue;
 321                  } else {
 322                      // Non numeric grade value supplied, possibly mapped wrong column.
 323                      $this->cleanup_import(get_string('badgrade', 'grades'));
 324                      return null;
 325                  }
 326              }
 327              $newgrade->finalgrade = $value;
 328          }
 329          $this->newgrades[] = $newgrade;
 330          return $this->newgrades;
 331      }
 332  
 333      /**
 334       * Clean up failed CSV grade import. Clears the temp table for inserting grades.
 335       *
 336       * @param string $notification The error message to display from the unsuccessful grade import.
 337       */
 338      protected function cleanup_import($notification) {
 339          $this->status = false;
 340          import_cleanup($this->importcode);
 341          $this->gradebookerrors[] = $notification;
 342      }
 343  
 344      /**
 345       * Check user mapping.
 346       *
 347       * @param string $mappingidentifier The user field that we are matching together.
 348       * @param string $value The value we are checking / importing.
 349       * @param array $header The column headers of the csv file.
 350       * @param array $map Mapping information provided by the user.
 351       * @param int $key Current row identifier.
 352       * @param int $courseid The course ID.
 353       * @param int $feedbackgradeid The ID of the grade item that the feedback relates to.
 354       * @param bool $verbosescales Form setting for grading with scales.
 355       */
 356      protected function map_user_data_with_value($mappingidentifier, $value, $header, $map, $key, $courseid, $feedbackgradeid,
 357              $verbosescales) {
 358  
 359          // Fields that the user can be mapped from.
 360          $userfields = array(
 361              'userid' => array(
 362                  'field' => 'id',
 363                  'label' => 'id',
 364              ),
 365              'useridnumber' => array(
 366                  'field' => 'idnumber',
 367                  'label' => 'idnumber',
 368              ),
 369              'useremail' => array(
 370                  'field' => 'email',
 371                  'label' => 'email address',
 372              ),
 373              'username' => array(
 374                  'field' => 'username',
 375                  'label' => 'username',
 376              ),
 377          );
 378  
 379          switch ($mappingidentifier) {
 380              case 'userid':
 381              case 'useridnumber':
 382              case 'useremail':
 383              case 'username':
 384                  // Skip invalid row with blank user field.
 385                  if (!empty($value)) {
 386                      $this->studentid = $this->check_user_exists($value, $userfields[$mappingidentifier]);
 387                  }
 388              break;
 389              case 'new':
 390                  $this->import_new_grade_item($header, $key, $value);
 391              break;
 392              case 'feedback':
 393                  if ($feedbackgradeid) {
 394                      $feedback = $this->create_feedback($courseid, $feedbackgradeid, $value);
 395                      if (isset($feedback)) {
 396                          $this->newfeedbacks[] = $feedback;
 397                      }
 398                  }
 399              break;
 400              default:
 401                  // Existing grade items.
 402                  if (!empty($map[$key])) {
 403                      $this->newgrades = $this->update_grade_item($courseid, $map, $key, $verbosescales, $value,
 404                              $mappingidentifier);
 405                  }
 406                  // Otherwise, we ignore this column altogether because user has chosen
 407                  // to ignore them (e.g. institution, address etc).
 408              break;
 409          }
 410      }
 411  
 412      /**
 413       * Checks and prepares grade data for inserting into the gradebook.
 414       *
 415       * @param array $header Column headers of the CSV file.
 416       * @param object $formdata Mapping information from the preview page.
 417       * @param object $csvimport csv import reader object for iterating over the imported CSV file.
 418       * @param int $courseid The course ID.
 419       * @param bool $separatemode If we have groups are they separate?
 420       * @param mixed $currentgroup current group information.
 421       * @param bool $verbosescales Form setting for grading with scales.
 422       * @return bool True if the status for importing is okay, false if there are errors.
 423       */
 424      public function prepare_import_grade_data($header, $formdata, $csvimport, $courseid, $separatemode, $currentgroup,
 425              $verbosescales) {
 426          global $DB, $USER;
 427  
 428          // The import code is used for inserting data into the grade tables.
 429          $this->importcode = $formdata->importcode;
 430          $this->status = true;
 431          $this->headers = $header;
 432          $this->studentid = null;
 433          $this->gradebookerrors = null;
 434          $forceimport = $formdata->forceimport;
 435          // Temporary array to keep track of what new headers are processed.
 436          $this->newgradeitems = array();
 437          $this->trim_headers();
 438          $timeexportkey = null;
 439          $map = array();
 440          // Loops mapping_0, mapping_1 .. mapping_n and construct $map array.
 441          foreach ($header as $i => $head) {
 442              if (isset($formdata->{'mapping_'.$i})) {
 443                  $map[$i] = $formdata->{'mapping_'.$i};
 444              }
 445              if ($head == get_string('timeexported', 'gradeexport_txt')) {
 446                  $timeexportkey = $i;
 447              }
 448          }
 449  
 450          // If mapping information is supplied.
 451          $map[clean_param($formdata->mapfrom, PARAM_RAW)] = clean_param($formdata->mapto, PARAM_RAW);
 452  
 453          // Check for mapto collisions.
 454          $maperrors = array();
 455          foreach ($map as $i => $j) {
 456              if ($j == 0) {
 457                  // You can have multiple ignores.
 458                  continue;
 459              } else {
 460                  if (!isset($maperrors[$j])) {
 461                      $maperrors[$j] = true;
 462                  } else {
 463                      // Collision.
 464                      print_error('cannotmapfield', '', '', $j);
 465                  }
 466              }
 467          }
 468  
 469          $this->raise_limits();
 470  
 471          $csvimport->init();
 472  
 473          while ($line = $csvimport->next()) {
 474              if (count($line) <= 1) {
 475                  // There is no data on this line, move on.
 476                  continue;
 477              }
 478  
 479              // Array to hold all grades to be inserted.
 480              $this->newgrades = array();
 481              // Array to hold all feedback.
 482              $this->newfeedbacks = array();
 483              // Each line is a student record.
 484              foreach ($line as $key => $value) {
 485  
 486                  $value = clean_param($value, PARAM_RAW);
 487                  $value = trim($value);
 488  
 489                  /*
 490                   * the options are
 491                   * 1) userid, useridnumber, usermail, username - used to identify user row
 492                   * 2) new - new grade item
 493                   * 3) id - id of the old grade item to map onto
 494                   * 3) feedback_id - feedback for grade item id
 495                   */
 496  
 497                  // Explode the mapping for feedback into a label 'feedback' and the identifying number.
 498                  $mappingbase = explode("_", $map[$key]);
 499                  $mappingidentifier = $mappingbase[0];
 500                  // Set the feedback identifier if it exists.
 501                  if (isset($mappingbase[1])) {
 502                      $feedbackgradeid = (int)$mappingbase[1];
 503                  } else {
 504                      $feedbackgradeid = '';
 505                  }
 506  
 507                  $this->map_user_data_with_value($mappingidentifier, $value, $header, $map, $key, $courseid, $feedbackgradeid,
 508                          $verbosescales);
 509                  if ($this->status === false) {
 510                      return $this->status;
 511                  }
 512              }
 513  
 514              // No user mapping supplied at all, or user mapping failed.
 515              if (empty($this->studentid) || !is_numeric($this->studentid)) {
 516                  // User not found, abort whole import.
 517                  $this->cleanup_import(get_string('usermappingerrorusernotfound', 'grades'));
 518                  break;
 519              }
 520  
 521              if ($separatemode and !groups_is_member($currentgroup, $this->studentid)) {
 522                  // Not allowed to import into this group, abort.
 523                  $this->cleanup_import(get_string('usermappingerrorcurrentgroup', 'grades'));
 524                  break;
 525              }
 526  
 527              // Insert results of this students into buffer.
 528              if ($this->status and !empty($this->newgrades)) {
 529  
 530                  foreach ($this->newgrades as $newgrade) {
 531  
 532                      // Check if grade_grade is locked and if so, abort.
 533                      if (!empty($newgrade->itemid) and $gradegrade = new grade_grade(array('itemid' => $newgrade->itemid,
 534                              'userid' => $this->studentid))) {
 535                          if ($gradegrade->is_locked()) {
 536                              // Individual grade locked.
 537                              $this->cleanup_import(get_string('gradelocked', 'grades'));
 538                              return $this->status;
 539                          }
 540                          // Check if the force import option is disabled and the last exported date column is present.
 541                          if (!$forceimport && !empty($timeexportkey)) {
 542                              $exportedtime = $line[$timeexportkey];
 543                              if (clean_param($exportedtime, PARAM_INT) != $exportedtime || $exportedtime > time() ||
 544                                      $exportedtime < strtotime("-1 year", time())) {
 545                                  // The date is invalid, or in the future, or more than a year old.
 546                                  $this->cleanup_import(get_string('invalidgradeexporteddate', 'grades'));
 547                                  return $this->status;
 548  
 549                              }
 550                              $timemodified = $gradegrade->get_dategraded();
 551                              if (!empty($timemodified) && ($exportedtime < $timemodified)) {
 552                                  // The item was graded after we exported it, we return here not to override it.
 553                                  $user = core_user::get_user($this->studentid);
 554                                  $this->cleanup_import(get_string('gradealreadyupdated', 'grades', fullname($user)));
 555                                  return $this->status;
 556                              }
 557                          }
 558                      }
 559                      $insertid = self::insert_grade_record($newgrade, $this->studentid);
 560                      // Check to see if the insert was successful.
 561                      if (empty($insertid)) {
 562                          return null;
 563                      }
 564                  }
 565              }
 566  
 567              // Updating/inserting all comments here.
 568              if ($this->status and !empty($this->newfeedbacks)) {
 569                  foreach ($this->newfeedbacks as $newfeedback) {
 570                      $sql = "SELECT *
 571                                FROM {grade_import_values}
 572                               WHERE importcode=? AND userid=? AND itemid=? AND importer=?";
 573                      if ($feedback = $DB->get_record_sql($sql, array($this->importcode, $this->studentid, $newfeedback->itemid,
 574                              $USER->id))) {
 575                          $newfeedback->id = $feedback->id;
 576                          $DB->update_record('grade_import_values', $newfeedback);
 577  
 578                      } else {
 579                          // The grade item for this is not updated.
 580                          $newfeedback->importonlyfeedback = true;
 581                          $insertid = self::insert_grade_record($newfeedback, $this->studentid);
 582                          // Check to see if the insert was successful.
 583                          if (empty($insertid)) {
 584                              return null;
 585                          }
 586                      }
 587                  }
 588              }
 589          }
 590          return $this->status;
 591      }
 592  
 593      /**
 594       * Returns the headers parameter for this class.
 595       *
 596       * @return array returns headers parameter for this class.
 597       */
 598      public function get_headers() {
 599          return $this->headers;
 600      }
 601  
 602      /**
 603       * Returns the error parameter for this class.
 604       *
 605       * @return string returns error parameter for this class.
 606       */
 607      public function get_error() {
 608          return $this->error;
 609      }
 610  
 611      /**
 612       * Returns the iid parameter for this class.
 613       *
 614       * @return int returns iid parameter for this class.
 615       */
 616      public function get_iid() {
 617          return $this->iid;
 618      }
 619  
 620      /**
 621       * Returns the preview_data parameter for this class.
 622       *
 623       * @return array returns previewdata parameter for this class.
 624       */
 625      public function get_previewdata() {
 626          return $this->previewdata;
 627      }
 628  
 629      /**
 630       * Returns the gradebookerrors parameter for this class.
 631       *
 632       * @return array returns gradebookerrors parameter for this class.
 633       */
 634      public function get_gradebookerrors() {
 635          return $this->gradebookerrors;
 636      }
 637  }