Change text tokens to proper tokens (instead of strings)

src/Mustache/Compiler.php

@@ -55,7 +55,7 @@ class Mustache_Compiler {
 		$code = '';
 		$level++;
 		foreach ($tree as $node) {
-			switch (is_string($node) ? 'text' : $node[Mustache_Tokenizer::TYPE]) {
+			switch ($node[Mustache_Tokenizer::TYPE]) {
 				case Mustache_Tokenizer::T_SECTION:
 					$code .= $this->section(
 						$node[Mustache_Tokenizer::NODES],
@@ -98,8 +98,8 @@ class Mustache_Compiler {
 					break;
 
 
-				case 'text':
-					$code .= $this->text($node, $level);
+				case Mustache_Tokenizer::T_TEXT:
+					$code .= $this->text($node[Mustache_Tokenizer::VALUE], $level);
 					break;
 
 				default:

src/Mustache/Parser.php

@@ -46,7 +46,7 @@ class Mustache_Parser {
 
 			if ($token === null) {
 				continue;
-			} elseif (is_array($token)) {
+			} else {
 				switch ($token[Mustache_Tokenizer::TYPE]) {
 					case Mustache_Tokenizer::T_SECTION:
 					case Mustache_Tokenizer::T_INVERTED:
@@ -72,8 +72,6 @@ class Mustache_Parser {
 						$nodes[] = $token;
 						break;
 				}
-			} else {
-				$nodes[] = $token;
 			}
 
 		} while ($tokens->valid());

src/Mustache/Tokenizer.php

@@ -32,6 +32,7 @@ class Mustache_Tokenizer {
 	const T_ESCAPED      = '_v';
 	const T_UNESCAPED    = '{';
 	const T_UNESCAPED_2  = '&';
+	const T_TEXT         = '_t';
 
 	// Valid token types
 	private static $tagTypes = array(
@@ -63,6 +64,7 @@ class Mustache_Tokenizer {
 	const END    = 'end';
 	const INDENT = 'indent';
 	const NODES  = 'nodes';
+	const VALUE  = 'value';
 
 	private $state;
 	private $tagType;
@@ -187,7 +189,7 @@ class Mustache_Tokenizer {
 	 */
 	private function flushBuffer() {
 		if (!empty($this->buffer)) {
-			$this->tokens[] = $this->buffer;
+			$this->tokens[] = array(self::TYPE  => self::T_TEXT, self::VALUE => $this->buffer);
 			$this->buffer   = '';
 		}
 	}
@@ -201,12 +203,12 @@ class Mustache_Tokenizer {
 		$tokensCount = count($this->tokens);
 		for ($j = $this->lineStart; $j < $tokensCount; $j++) {
 			$token = $this->tokens[$j];
-			if (is_array($token) && isset(self::$tagTypes[$token[self::TYPE]])) {
+			if (isset(self::$tagTypes[$token[self::TYPE]])) {
 				if (isset(self::$interpolatedTags[$token[self::TYPE]])) {
 					return false;
 				}
-			} elseif (is_string($token)) {
-				if (preg_match('/\S/', $token)) {
+			} elseif ($token[self::TYPE] == self::T_TEXT) {
+				if (preg_match('/\S/', $token[self::VALUE])) {
 					return false;
 				}
 			}
@@ -225,16 +227,16 @@ class Mustache_Tokenizer {
 		if ($this->seenTag && $this->lineIsWhitespace()) {
 			$tokensCount = count($this->tokens);
 			for ($j = $this->lineStart; $j < $tokensCount; $j++) {
-				if (!is_array($this->tokens[$j])) {
-					if (isset($this->tokens[$j+1]) && is_array($this->tokens[$j+1]) && $this->tokens[$j+1][self::TYPE] == self::T_PARTIAL) {
-						$this->tokens[$j+1][self::INDENT] = (string) $this->tokens[$j];
+				if ($this->tokens[$j][self::TYPE] == self::T_TEXT) {
+					if (isset($this->tokens[$j+1]) && $this->tokens[$j+1][self::TYPE] == self::T_PARTIAL) {
+						$this->tokens[$j+1][self::INDENT] = $this->tokens[$j][self::VALUE];
 					}
 
 					$this->tokens[$j] = null;
 				}
 			}
 		} elseif (!$noNewLine) {
-			$this->tokens[] = "\n";
+			$this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => "\n");
 		}
 
 		$this->seenTag   = false;

test/Mustache/Test/CompilerTest.php

@@ -34,14 +34,14 @@ class Mustache_Test_CompilerTest extends \PHPUnit_Framework_TestCase {
 				'return $buffer;',
 			)),
 
-			array('', array('TEXT'), 'Monkey', false, 'UTF-8', array(
+			array('', array($this->createTextToken('TEXT')), 'Monkey', false, 'UTF-8', array(
 				"\nclass Monkey extends Mustache_Template",
 				'return htmlspecialchars($buffer, ENT_COMPAT, \'UTF-8\');',
 				'$buffer .= $indent . \'TEXT\';',
 				'return $buffer;',
 			)),
 
-			array('', array('TEXT'), 'Monkey', true, 'ISO-8859-1', array(
+			array('', array($this->createTextToken('TEXT')), 'Monkey', true, 'ISO-8859-1', array(
 				"\nclass Monkey extends Mustache_Template",
 				'$buffer .= $indent . \'TEXT\';',
 				'return call_user_func($this->mustache->getEscape(), $buffer);',
@@ -51,8 +51,8 @@ class Mustache_Test_CompilerTest extends \PHPUnit_Framework_TestCase {
 			array(
 				'',
 				array(
-					'foo',
-					"\n",
+					$this->createTextToken('foo'),
+					$this->createTextToken("\n"),
 					array(
 						Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
 						Mustache_Tokenizer::NAME => 'name',
@@ -61,7 +61,7 @@ class Mustache_Test_CompilerTest extends \PHPUnit_Framework_TestCase {
 						Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
 						Mustache_Tokenizer::NAME => '.',
 					),
-					"'bar'",
+					$this->createTextToken("'bar'"),
 				),
 				'Monkey',
 				false,
@@ -88,4 +88,11 @@ class Mustache_Test_CompilerTest extends \PHPUnit_Framework_TestCase {
 		$compiler = new Mustache_Compiler;
 		$compiler->compile('', array(array(Mustache_Tokenizer::TYPE => 'invalid')), 'SomeClass');
 	}
+
+	private function createTextToken($value) {
+		return array(
+			Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+			Mustache_Tokenizer::VALUE => $value,
+		);
+	}
 }

test/Mustache/Test/ParserTest.php

@@ -32,8 +32,14 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase {
 			),
 
 			array(
-				array('text'),
-				array('text')
+				array(array(
+					Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+					Mustache_Tokenizer::VALUE => 'text'
+				)),
+				array(array(
+					Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+					Mustache_Tokenizer::VALUE => 'text'
+				)),
 			),
 
 			array(
@@ -49,7 +55,10 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase {
 
 			array(
 				array(
-					'foo',
+					array(
+						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+						Mustache_Tokenizer::VALUE => 'foo'
+					),
 					array(
 						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_INVERTED,
 						Mustache_Tokenizer::INDEX => 123,
@@ -64,10 +73,16 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase {
 						Mustache_Tokenizer::INDEX => 456,
 						Mustache_Tokenizer::NAME  => 'parent'
 					),
-					'bar',
+					array(
+						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+						Mustache_Tokenizer::VALUE => 'bar'
+					),
 				),
 				array(
-					'foo',
+					array(
+						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+						Mustache_Tokenizer::VALUE => 'foo'
+					),
 					array(
 						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_INVERTED,
 						Mustache_Tokenizer::NAME  => 'parent',
@@ -80,7 +95,10 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase {
 							),
 						),
 					),
-					'bar',
+					array(
+						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+						Mustache_Tokenizer::VALUE => 'bar'
+					),
 				),
 			),
 

test/Mustache/Test/TokenizerTest.php

@@ -27,13 +27,23 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase {
 			array(
 				'text',
 				null,
-				array('text'),
+				array(
+					array(
+						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+						Mustache_Tokenizer::VALUE => 'text',
+					),
+				),
 			),
 
 			array(
 				'text',
 				'<<< >>>',
-				array('text'),
+				array(
+					array(
+						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+						Mustache_Tokenizer::VALUE => 'text',
+					),
+				),
 			),
 
 			array(
@@ -53,7 +63,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase {
 			array(
 				'{{ name }}',
 				'<<< >>>',
-				array('{{ name }}'),
+				array(
+					array(
+						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+						Mustache_Tokenizer::VALUE => '{{ name }}',
+					),
+				),
 			),
 
 			array(
@@ -81,7 +96,10 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase {
 						Mustache_Tokenizer::CTAG  => '}}',
 						Mustache_Tokenizer::INDEX => 8,
 					),
-					"\n",
+					array(
+						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+						Mustache_Tokenizer::VALUE => "\n",
+					),
 					array(
 						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_SECTION,
 						Mustache_Tokenizer::NAME  => 'b',
@@ -104,7 +122,10 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase {
 						Mustache_Tokenizer::CTAG  => '|',
 						Mustache_Tokenizer::INDEX => 37,
 					),
-					"\n",
+					array(
+						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+						Mustache_Tokenizer::VALUE => "\n",
+					),
 					array(
 						Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_UNESCAPED,
 						Mustache_Tokenizer::NAME  => 'd',