[Summary view] [Print] [Text view]

   1  <?php
   2  
   3  /*
   4   * This file is part of Mustache.php.
   5   *
   6   * (c) 2010-2015 Justin Hileman
   7   *
   8   * For the full copyright and license information, please view the LICENSE
   9   * file that was distributed with this source code.
  10   */
  11  
  12  /**
  13   * Mustache Tokenizer class.
  14   *
  15   * This class is responsible for turning raw template source into a set of Mustache tokens.
  16   */
  17  class Mustache_Tokenizer
  18  {
  19      // Finite state machine states
  20      const IN_TEXT     = 0;
  21      const IN_TAG_TYPE = 1;
  22      const IN_TAG      = 2;
  23  
  24      // Token types
  25      const T_SECTION      = '#';
  26      const T_INVERTED     = '^';
  27      const T_END_SECTION  = '/';
  28      const T_COMMENT      = '!';
  29      const T_PARTIAL      = '>';
  30      const T_PARENT       = '<';
  31      const T_DELIM_CHANGE = '=';
  32      const T_ESCAPED      = '_v';
  33      const T_UNESCAPED    = '{';
  34      const T_UNESCAPED_2  = '&';
  35      const T_TEXT         = '_t';
  36      const T_PRAGMA       = '%';
  37      const T_BLOCK_VAR    = '$';
  38      const T_BLOCK_ARG    = '$arg';
  39  
  40      // Valid token types
  41      private static $tagTypes = array(
  42          self::T_SECTION      => true,
  43          self::T_INVERTED     => true,
  44          self::T_END_SECTION  => true,
  45          self::T_COMMENT      => true,
  46          self::T_PARTIAL      => true,
  47          self::T_PARENT       => true,
  48          self::T_DELIM_CHANGE => true,
  49          self::T_ESCAPED      => true,
  50          self::T_UNESCAPED    => true,
  51          self::T_UNESCAPED_2  => true,
  52          self::T_PRAGMA       => true,
  53          self::T_BLOCK_VAR    => true,
  54      );
  55  
  56      // Token properties
  57      const TYPE    = 'type';
  58      const NAME    = 'name';
  59      const OTAG    = 'otag';
  60      const CTAG    = 'ctag';
  61      const LINE    = 'line';
  62      const INDEX   = 'index';
  63      const END     = 'end';
  64      const INDENT  = 'indent';
  65      const NODES   = 'nodes';
  66      const VALUE   = 'value';
  67      const FILTERS = 'filters';
  68  
  69      private $state;
  70      private $tagType;
  71      private $buffer;
  72      private $tokens;
  73      private $seenTag;
  74      private $line;
  75      private $otag;
  76      private $ctag;
  77      private $otagLen;
  78      private $ctagLen;
  79  
  80      /**
  81       * Scan and tokenize template source.
  82       *
  83       * @throws Mustache_Exception_SyntaxException when mismatched section tags are encountered.
  84       *
  85       * @param string $text       Mustache template source to tokenize
  86       * @param string $delimiters Optionally, pass initial opening and closing delimiters (default: null)
  87       *
  88       * @return array Set of Mustache tokens
  89       */
  90      public function scan($text, $delimiters = null)
  91      {
  92          // Setting mbstring.func_overload makes things *really* slow.
  93          // Let's do everyone a favor and scan this string as ASCII instead.
  94          $encoding = null;
  95          if (function_exists('mb_internal_encoding') && ini_get('mbstring.func_overload') & 2) {
  96              $encoding = mb_internal_encoding();
  97              mb_internal_encoding('ASCII');
  98          }
  99  
 100          $this->reset();
 101  
 102          if ($delimiters = trim($delimiters)) {
 103              $this->setDelimiters($delimiters);
 104          }
 105  
 106          $len = strlen($text);
 107          for ($i = 0; $i < $len; $i++) {
 108              switch ($this->state) {
 109                  case self::IN_TEXT:
 110                      if ($this->tagChange($this->otag, $this->otagLen, $text, $i)) {
 111                          $i--;
 112                          $this->flushBuffer();
 113                          $this->state = self::IN_TAG_TYPE;
 114                      } else {
 115                          $char = $text[$i];
 116                          $this->buffer .= $char;
 117                          if ($char === "\n") {
 118                              $this->flushBuffer();
 119                              $this->line++;
 120                          }
 121                      }
 122                      break;
 123  
 124                  case self::IN_TAG_TYPE:
 125                      $i += $this->otagLen - 1;
 126                      $char = $text[$i + 1];
 127                      if (isset(self::$tagTypes[$char])) {
 128                          $tag = $char;
 129                          $this->tagType = $tag;
 130                      } else {
 131                          $tag = null;
 132                          $this->tagType = self::T_ESCAPED;
 133                      }
 134  
 135                      if ($this->tagType === self::T_DELIM_CHANGE) {
 136                          $i = $this->changeDelimiters($text, $i);
 137                          $this->state = self::IN_TEXT;
 138                      } elseif ($this->tagType === self::T_PRAGMA) {
 139                          $i = $this->addPragma($text, $i);
 140                          $this->state = self::IN_TEXT;
 141                      } else {
 142                          if ($tag !== null) {
 143                              $i++;
 144                          }
 145                          $this->state = self::IN_TAG;
 146                      }
 147                      $this->seenTag = $i;
 148                      break;
 149  
 150                  default:
 151                      if ($this->tagChange($this->ctag, $this->ctagLen, $text, $i)) {
 152                          $token = array(
 153                              self::TYPE  => $this->tagType,
 154                              self::NAME  => trim($this->buffer),
 155                              self::OTAG  => $this->otag,
 156                              self::CTAG  => $this->ctag,
 157                              self::LINE  => $this->line,
 158                              self::INDEX => ($this->tagType === self::T_END_SECTION) ? $this->seenTag - $this->otagLen : $i + $this->ctagLen,
 159                          );
 160  
 161                          if ($this->tagType === self::T_UNESCAPED) {
 162                              // Clean up `{{{ tripleStache }}}` style tokens.
 163                              if ($this->ctag === '}}') {
 164                                  if (($i + 2 < $len) && $text[$i + 2] === '}') {
 165                                      $i++;
 166                                  } else {
 167                                      $msg = sprintf(
 168                                          'Mismatched tag delimiters: %s on line %d',
 169                                          $token[self::NAME],
 170                                          $token[self::LINE]
 171                                      );
 172  
 173                                      throw new Mustache_Exception_SyntaxException($msg, $token);
 174                                  }
 175                              } else {
 176                                  $lastName = $token[self::NAME];
 177                                  if (substr($lastName, -1) === '}') {
 178                                      $token[self::NAME] = trim(substr($lastName, 0, -1));
 179                                  } else {
 180                                      $msg = sprintf(
 181                                          'Mismatched tag delimiters: %s on line %d',
 182                                          $token[self::NAME],
 183                                          $token[self::LINE]
 184                                      );
 185  
 186                                      throw new Mustache_Exception_SyntaxException($msg, $token);
 187                                  }
 188                              }
 189                          }
 190  
 191                          $this->buffer = '';
 192                          $i += $this->ctagLen - 1;
 193                          $this->state = self::IN_TEXT;
 194                          $this->tokens[] = $token;
 195                      } else {
 196                          $this->buffer .= $text[$i];
 197                      }
 198                      break;
 199              }
 200          }
 201  
 202          $this->flushBuffer();
 203  
 204          // Restore the user's encoding...
 205          if ($encoding) {
 206              mb_internal_encoding($encoding);
 207          }
 208  
 209          return $this->tokens;
 210      }
 211  
 212      /**
 213       * Helper function to reset tokenizer internal state.
 214       */
 215      private function reset()
 216      {
 217          $this->state   = self::IN_TEXT;
 218          $this->tagType = null;
 219          $this->buffer  = '';
 220          $this->tokens  = array();
 221          $this->seenTag = false;
 222          $this->line    = 0;
 223          $this->otag    = '{{';
 224          $this->ctag    = '}}';
 225          $this->otagLen = 2;
 226          $this->ctagLen = 2;
 227      }
 228  
 229      /**
 230       * Flush the current buffer to a token.
 231       */
 232      private function flushBuffer()
 233      {
 234          if (strlen($this->buffer) > 0) {
 235              $this->tokens[] = array(
 236                  self::TYPE  => self::T_TEXT,
 237                  self::LINE  => $this->line,
 238                  self::VALUE => $this->buffer,
 239              );
 240              $this->buffer   = '';
 241          }
 242      }
 243  
 244      /**
 245       * Change the current Mustache delimiters. Set new `otag` and `ctag` values.
 246       *
 247       * @param string $text  Mustache template source
 248       * @param int    $index Current tokenizer index
 249       *
 250       * @return int New index value
 251       */
 252      private function changeDelimiters($text, $index)
 253      {
 254          $startIndex = strpos($text, '=', $index) + 1;
 255          $close      = '=' . $this->ctag;
 256          $closeIndex = strpos($text, $close, $index);
 257  
 258          $this->setDelimiters(trim(substr($text, $startIndex, $closeIndex - $startIndex)));
 259  
 260          $this->tokens[] = array(
 261              self::TYPE => self::T_DELIM_CHANGE,
 262              self::LINE => $this->line,
 263          );
 264  
 265          return $closeIndex + strlen($close) - 1;
 266      }
 267  
 268      /**
 269       * Set the current Mustache `otag` and `ctag` delimiters.
 270       *
 271       * @param string $delimiters
 272       */
 273      private function setDelimiters($delimiters)
 274      {
 275          list($otag, $ctag) = explode(' ', $delimiters);
 276          $this->otag = $otag;
 277          $this->ctag = $ctag;
 278          $this->otagLen = strlen($otag);
 279          $this->ctagLen = strlen($ctag);
 280      }
 281  
 282      /**
 283       * Add pragma token.
 284       *
 285       * Pragmas are hoisted to the front of the template, so all pragma tokens
 286       * will appear at the front of the token list.
 287       *
 288       * @param string $text
 289       * @param int    $index
 290       *
 291       * @return int New index value
 292       */
 293      private function addPragma($text, $index)
 294      {
 295          $end    = strpos($text, $this->ctag, $index);
 296          $pragma = trim(substr($text, $index + 2, $end - $index - 2));
 297  
 298          // Pragmas are hoisted to the front of the template.
 299          array_unshift($this->tokens, array(
 300              self::TYPE => self::T_PRAGMA,
 301              self::NAME => $pragma,
 302              self::LINE => 0,
 303          ));
 304  
 305          return $end + $this->ctagLen - 1;
 306      }
 307  
 308      /**
 309       * Test whether it's time to change tags.
 310       *
 311       * @param string $tag    Current tag name
 312       * @param int    $tagLen Current tag name length
 313       * @param string $text   Mustache template source
 314       * @param int    $index  Current tokenizer index
 315       *
 316       * @return bool True if this is a closing section tag
 317       */
 318      private function tagChange($tag, $tagLen, $text, $index)
 319      {
 320          return substr($text, $index, $tagLen) === $tag;
 321      }
 322  }