Framework/Mustache
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change text tokens to proper tokens (instead of strings)

Browse Source
This commit is contained in:
Justin Hileman
2012-03-26 19:34:55 -07:00
parent 1430b27f86
commit 2328615772
6 changed files with 76 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Mustache/Compiler.php
+3 -3
View File
@@ -55,7 +55,7 @@ class Mustache_Compiler {
$code = ''; $code = '';
$level++; $level++;
foreach ($tree as $node) { foreach ($tree as $node) {
switch (is_string($node) ? 'text' : $node[Mustache_Tokenizer::TYPE]) { switch ($node[Mustache_Tokenizer::TYPE]) {
case Mustache_Tokenizer::T_SECTION: case Mustache_Tokenizer::T_SECTION:
$code .= $this->section( $code .= $this->section(
$node[Mustache_Tokenizer::NODES], $node[Mustache_Tokenizer::NODES],
@@ -98,8 +98,8 @@ class Mustache_Compiler {
break; break;
case 'text': case Mustache_Tokenizer::T_TEXT:
$code .= $this->text($node, $level); $code .= $this->text($node[Mustache_Tokenizer::VALUE], $level);
break; break;
default: default:
src/Mustache/Parser.php
+1 -3
View File
@@ -46,7 +46,7 @@ class Mustache_Parser {
if ($token === null) { if ($token === null) {
continue; continue;
} elseif (is_array($token)) { } else {
switch ($token[Mustache_Tokenizer::TYPE]) { switch ($token[Mustache_Tokenizer::TYPE]) {
case Mustache_Tokenizer::T_SECTION: case Mustache_Tokenizer::T_SECTION:
case Mustache_Tokenizer::T_INVERTED: case Mustache_Tokenizer::T_INVERTED:
@@ -72,8 +72,6 @@ class Mustache_Parser {
$nodes[] = $token; $nodes[] = $token;
break; break;
} }
} else {
$nodes[] = $token;
} }
} while ($tokens->valid()); } while ($tokens->valid());
src/Mustache/Tokenizer.php
+10 -8
View File
@@ -32,6 +32,7 @@ class Mustache_Tokenizer {
const T_ESCAPED = '_v'; const T_ESCAPED = '_v';
const T_UNESCAPED = '{'; const T_UNESCAPED = '{';
const T_UNESCAPED_2 = '&'; const T_UNESCAPED_2 = '&';
const T_TEXT = '_t';
// Valid token types // Valid token types
private static $tagTypes = array( private static $tagTypes = array(
@@ -63,6 +64,7 @@ class Mustache_Tokenizer {
const END = 'end'; const END = 'end';
const INDENT = 'indent'; const INDENT = 'indent';
const NODES = 'nodes'; const NODES = 'nodes';
const VALUE = 'value';
private $state; private $state;
private $tagType; private $tagType;
@@ -187,7 +189,7 @@ class Mustache_Tokenizer {
*/ */
private function flushBuffer() { private function flushBuffer() {
if (!empty($this->buffer)) { if (!empty($this->buffer)) {
$this->tokens[] = $this->buffer; $this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => $this->buffer);
$this->buffer = ''; $this->buffer = '';
} }
} }
@@ -201,12 +203,12 @@ class Mustache_Tokenizer {
$tokensCount = count($this->tokens); $tokensCount = count($this->tokens);
for ($j = $this->lineStart; $j < $tokensCount; $j++) { for ($j = $this->lineStart; $j < $tokensCount; $j++) {
$token = $this->tokens[$j]; $token = $this->tokens[$j];
if (is_array($token) && isset(self::$tagTypes[$token[self::TYPE]])) { if (isset(self::$tagTypes[$token[self::TYPE]])) {
if (isset(self::$interpolatedTags[$token[self::TYPE]])) { if (isset(self::$interpolatedTags[$token[self::TYPE]])) {
return false; return false;
} }
} elseif (is_string($token)) { } elseif ($token[self::TYPE] == self::T_TEXT) {
if (preg_match('/\S/', $token)) { if (preg_match('/\S/', $token[self::VALUE])) {
return false; return false;
} }
} }
@@ -225,16 +227,16 @@ class Mustache_Tokenizer {
if ($this->seenTag && $this->lineIsWhitespace()) { if ($this->seenTag && $this->lineIsWhitespace()) {
$tokensCount = count($this->tokens); $tokensCount = count($this->tokens);
for ($j = $this->lineStart; $j < $tokensCount; $j++) { for ($j = $this->lineStart; $j < $tokensCount; $j++) {
if (!is_array($this->tokens[$j])) { if ($this->tokens[$j][self::TYPE] == self::T_TEXT) {
if (isset($this->tokens[$j+1]) && is_array($this->tokens[$j+1]) && $this->tokens[$j+1][self::TYPE] == self::T_PARTIAL) { if (isset($this->tokens[$j+1]) && $this->tokens[$j+1][self::TYPE] == self::T_PARTIAL) {
$this->tokens[$j+1][self::INDENT] = (string) $this->tokens[$j]; $this->tokens[$j+1][self::INDENT] = $this->tokens[$j][self::VALUE];
} }
$this->tokens[$j] = null; $this->tokens[$j] = null;
} }
} }
} elseif (!$noNewLine) { } elseif (!$noNewLine) {
$this->tokens[] = "\n"; $this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => "\n");
} }
$this->seenTag = false; $this->seenTag = false;
test/Mustache/Test/CompilerTest.php
+12 -5
View File
@@ -34,14 +34,14 @@ class Mustache_Test_CompilerTest extends \PHPUnit_Framework_TestCase {
'return $buffer;', 'return $buffer;',
)), )),
array('', array('TEXT'), 'Monkey', false, 'UTF-8', array( array('', array($this->createTextToken('TEXT')), 'Monkey', false, 'UTF-8', array(
"\nclass Monkey extends Mustache_Template", "\nclass Monkey extends Mustache_Template",
'return htmlspecialchars($buffer, ENT_COMPAT, \'UTF-8\');', 'return htmlspecialchars($buffer, ENT_COMPAT, \'UTF-8\');',
'$buffer .= $indent . \'TEXT\';', '$buffer .= $indent . \'TEXT\';',
'return $buffer;', 'return $buffer;',
)), )),
array('', array('TEXT'), 'Monkey', true, 'ISO-8859-1', array( array('', array($this->createTextToken('TEXT')), 'Monkey', true, 'ISO-8859-1', array(
"\nclass Monkey extends Mustache_Template", "\nclass Monkey extends Mustache_Template",
'$buffer .= $indent . \'TEXT\';', '$buffer .= $indent . \'TEXT\';',
'return call_user_func($this->mustache->getEscape(), $buffer);', 'return call_user_func($this->mustache->getEscape(), $buffer);',
@@ -51,8 +51,8 @@ class Mustache_Test_CompilerTest extends \PHPUnit_Framework_TestCase {
array( array(
'', '',
array( array(
'foo', $this->createTextToken('foo'),
"\n", $this->createTextToken("\n"),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::NAME => 'name', Mustache_Tokenizer::NAME => 'name',
@@ -61,7 +61,7 @@ class Mustache_Test_CompilerTest extends \PHPUnit_Framework_TestCase {
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::NAME => '.', Mustache_Tokenizer::NAME => '.',
), ),
"'bar'", $this->createTextToken("'bar'"),
), ),
'Monkey', 'Monkey',
false, false,
@@ -88,4 +88,11 @@ class Mustache_Test_CompilerTest extends \PHPUnit_Framework_TestCase {
$compiler = new Mustache_Compiler; $compiler = new Mustache_Compiler;
$compiler->compile('', array(array(Mustache_Tokenizer::TYPE => 'invalid')), 'SomeClass'); $compiler->compile('', array(array(Mustache_Tokenizer::TYPE => 'invalid')), 'SomeClass');
} }
private function createTextToken($value) {
return array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => $value,
);
}
} }
test/Mustache/Test/ParserTest.php
+24 -6
View File
@@ -32,8 +32,14 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase {
), ),
array( array(
array('text'), array(array(
array('text') Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'text'
)),
array(array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'text'
)),
), ),
array( array(
@@ -49,7 +55,10 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase {
array( array(
array( array(
'foo', array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'foo'
),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
@@ -64,10 +73,16 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase {
Mustache_Tokenizer::INDEX => 456, Mustache_Tokenizer::INDEX => 456,
Mustache_Tokenizer::NAME => 'parent' Mustache_Tokenizer::NAME => 'parent'
), ),
'bar', array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'bar'
),
), ),
array( array(
'foo', array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'foo'
),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::NAME => 'parent', Mustache_Tokenizer::NAME => 'parent',
@@ -80,7 +95,10 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase {
), ),
), ),
), ),
'bar', array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'bar'
),
), ),
), ),
test/Mustache/Test/TokenizerTest.php
+26 -5
View File
@@ -27,13 +27,23 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase {
array( array(
'text', 'text',
null, null,
array('text'), array(
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'text',
),
),
), ),
array( array(
'text', 'text',
'<<< >>>', '<<< >>>',
array('text'), array(
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'text',
),
),
), ),
array( array(
@@ -53,7 +63,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase {
array( array(
'{{ name }}', '{{ name }}',
'<<< >>>', '<<< >>>',
array('{{ name }}'), array(
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => '{{ name }}',
),
),
), ),
array( array(
@@ -81,7 +96,10 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase {
Mustache_Tokenizer::CTAG => '}}', Mustache_Tokenizer::CTAG => '}}',
Mustache_Tokenizer::INDEX => 8, Mustache_Tokenizer::INDEX => 8,
), ),
"\n", array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => "\n",
),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,
Mustache_Tokenizer::NAME => 'b', Mustache_Tokenizer::NAME => 'b',
@@ -104,7 +122,10 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase {
Mustache_Tokenizer::CTAG => '|', Mustache_Tokenizer::CTAG => '|',
Mustache_Tokenizer::INDEX => 37, Mustache_Tokenizer::INDEX => 37,
), ),
"\n", array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => "\n",
),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_UNESCAPED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_UNESCAPED,
Mustache_Tokenizer::NAME => 'd', Mustache_Tokenizer::NAME => 'd',