Framework/Mustache
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/whitespace-tokenizing' into dev

Browse Source
This commit is contained in:
Justin Hileman
2013-06-25 08:43:00 -07:00
parent 4112bcc3f7 87a734e60c
commit 287d2f74a8
6 changed files with 220 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Mustache/Compiler.php
+9 -11
View File
@@ -373,13 +373,11 @@ class Mustache_Compiler
*/
private function text($text, $level)
{
if ($text === "\n") {
$this->indentNextLine = true;
$indentNextLine = (substr($text, -1) === "\n");
$code = sprintf($this->prepare(self::TEXT, $level), $this->flushIndent(), var_export($text, true));
$this->indentNextLine = $indentNextLine;
return $this->prepare(self::LINE, $level);
} else {
return sprintf($this->prepare(self::TEXT, $level), $this->flushIndent(), var_export($text, true));
}
return $code;
}
/**
@@ -467,12 +465,12 @@ class Mustache_Compiler
*/
private function flushIndent()
{
if ($this->indentNextLine) {
$this->indentNextLine = false;
return self::LINE_INDENT;
} else {
if (!$this->indentNextLine) {
return '';
}
$this->indentNextLine = false;
return self::LINE_INDENT;
}
}
src/Mustache/Parser.php
+135 -30
View File
@@ -16,6 +16,8 @@
*/
class Mustache_Parser
{
private $lineNum;
private $lineTokens;
/**
* Process an array of Mustache tokens and convert them into a parse tree.
@@ -26,6 +28,9 @@ class Mustache_Parser
*/
public function parse(array $tokens = array())
{
$this->lineNum = -1;
$this->lineTokens = 0;
return $this->buildTree($tokens);
}
@@ -46,38 +51,62 @@ class Mustache_Parser
while (!empty($tokens)) {
$token = array_shift($tokens);
if ($token === null) {
continue;
if ($token[Mustache_Tokenizer::LINE] === $this->lineNum) {
$this->lineTokens++;
} else {
switch ($token[Mustache_Tokenizer::TYPE]) {
case Mustache_Tokenizer::T_SECTION:
case Mustache_Tokenizer::T_INVERTED:
$nodes[] = $this->buildTree($tokens, $token);
break;
case Mustache_Tokenizer::T_END_SECTION:
if (!isset($parent)) {
$msg = sprintf('Unexpected closing tag: /%s', $token[Mustache_Tokenizer::NAME]);
throw new Mustache_Exception_SyntaxException($msg, $token);
}
if ($token[Mustache_Tokenizer::NAME] !== $parent[Mustache_Tokenizer::NAME]) {
$msg = sprintf('Nesting error: %s vs. %s', $parent[Mustache_Tokenizer::NAME], $token[Mustache_Tokenizer::NAME]);
throw new Mustache_Exception_SyntaxException($msg, $token);
}
$parent[Mustache_Tokenizer::END] = $token[Mustache_Tokenizer::INDEX];
$parent[Mustache_Tokenizer::NODES] = $nodes;
return $parent;
break;
default:
$nodes[] = $token;
break;
}
$this->lineNum = $token[Mustache_Tokenizer::LINE];
$this->lineTokens = 0;
}
};
switch ($token[Mustache_Tokenizer::TYPE]) {
case Mustache_Tokenizer::T_DELIM_CHANGE:
$this->clearStandaloneLines($nodes, $tokens);
break;
case Mustache_Tokenizer::T_SECTION:
case Mustache_Tokenizer::T_INVERTED:
$this->clearStandaloneLines($nodes, $tokens);
$nodes[] = $this->buildTree($tokens, $token);
break;
case Mustache_Tokenizer::T_END_SECTION:
if (!isset($parent)) {
$msg = sprintf('Unexpected closing tag: /%s', $token[Mustache_Tokenizer::NAME]);
throw new Mustache_Exception_SyntaxException($msg, $token);
}
if ($token[Mustache_Tokenizer::NAME] !== $parent[Mustache_Tokenizer::NAME]) {
$msg = sprintf('Nesting error: %s vs. %s', $parent[Mustache_Tokenizer::NAME], $token[Mustache_Tokenizer::NAME]);
throw new Mustache_Exception_SyntaxException($msg, $token);
}
$this->clearStandaloneLines($nodes, $tokens);
$parent[Mustache_Tokenizer::END] = $token[Mustache_Tokenizer::INDEX];
$parent[Mustache_Tokenizer::NODES] = $nodes;
return $parent;
break;
case Mustache_Tokenizer::T_PARTIAL:
case Mustache_Tokenizer::T_PARTIAL_2:
// store the whitespace prefix for laters!
if ($indent = $this->clearStandaloneLines($nodes, $tokens)) {
$token[Mustache_Tokenizer::INDENT] = $indent[Mustache_Tokenizer::VALUE];
}
$nodes[] = $token;
break;
case Mustache_Tokenizer::T_PRAGMA:
case Mustache_Tokenizer::T_COMMENT:
$this->clearStandaloneLines($nodes, $tokens);
$nodes[] = $token;
break;
default:
$nodes[] = $token;
break;
}
}
if (isset($parent)) {
$msg = sprintf('Missing closing tag: %s', $parent[Mustache_Tokenizer::NAME]);
@@ -86,4 +115,80 @@ class Mustache_Parser
return $nodes;
}
/**
* Clear standalone line tokens.
*
* Returns a whitespace token for indenting partials, if applicable.
*
* @param array $nodes Parsed nodes.
* @param array $tokens Tokens to be parsed.
*
* @return array Resulting indent token, if any.
*/
private function clearStandaloneLines(array &$nodes, array &$tokens)
{
if ($this->lineTokens > 1) {
// this is the third or later node on this line, so it can't be standalone
return;
}
$prev = null;
if ($this->lineTokens === 1) {
// this is the second node on this line, so it can't be standalone
// unless the previous node is whitespace.
if ($prev = end($nodes)) {
if (!$this->tokenIsWhitespace($prev)) {
return;
}
}
}
$next = null;
if ($next = reset($tokens)) {
// If we're on a new line, bail.
if ($next[Mustache_Tokenizer::LINE] !== $this->lineNum) {
return;
}
// If the next token isn't whitespace, bail.
if (!$this->tokenIsWhitespace($next)) {
return;
}
if (count($tokens) !== 1) {
// Unless it's the last token in the template, the next token
// must end in newline for this to be standalone.
if (substr($next[Mustache_Tokenizer::VALUE], -1) !== "\n") {
return;
}
}
// Discard the whitespace suffix
array_shift($tokens);
}
if ($prev) {
// Return the whitespace prefix, if any
return array_pop($nodes);
}
}
/**
* Check whether token is a whitespace token.
*
* True if token type is T_TEXT and value is all whitespace characters.
*
* @param array $token
*
* @return boolean True if token is a whitespace token
*/
private function tokenIsWhitespace(array $token)
{
if ($token[Mustache_Tokenizer::TYPE] == Mustache_Tokenizer::T_TEXT) {
return preg_match('/^\s*$/', $token[Mustache_Tokenizer::VALUE]);
}
return false;
}
}
src/Mustache/Tokenizer.php
+31 -72
View File
@@ -63,20 +63,20 @@ class Mustache_Tokenizer
const NAME = 'name';
const OTAG = 'otag';
const CTAG = 'ctag';
const LINE = 'line';
const INDEX = 'index';
const END = 'end';
const INDENT = 'indent';
const NODES = 'nodes';
const VALUE = 'value';
private $pragmas;
private $state;
private $tagType;
private $tag;
private $buffer;
private $tokens;
private $seenTag;
private $lineStart;
private $line;
private $otag;
private $ctag;
@@ -108,16 +108,15 @@ class Mustache_Tokenizer
$this->state = self::IN_TAG_TYPE;
} else {
$char = substr($text, $i, 1);
$this->buffer .= $char;
if ($char == "\n") {
$this->filterLine();
} else {
$this->buffer .= $char;
$this->flushBuffer();
$this->line++;
}
}
break;
case self::IN_TAG_TYPE:
$i += strlen($this->otag) - 1;
$char = substr($text, $i + 1, 1);
if (isset(self::$tagTypes[$char])) {
@@ -150,6 +149,7 @@ class Mustache_Tokenizer
self::NAME => trim($this->buffer),
self::OTAG => $this->otag,
self::CTAG => $this->ctag,
self::LINE => $this->line,
self::INDEX => ($this->tagType == self::T_END_SECTION) ? $this->seenTag - strlen($this->otag) : $i + strlen($this->ctag)
);
@@ -174,15 +174,7 @@ class Mustache_Tokenizer
}
}
$this->filterLine(true);
// Pragmas are hoisted to the front of the template.
foreach ($this->pragmas as $pragma) {
array_unshift($this->tokens, array(
self::TYPE => self::T_PRAGMA,
self::NAME => $pragma,
));
}
$this->flushBuffer();
return $this->tokens;
}
@@ -198,10 +190,9 @@ class Mustache_Tokenizer
$this->buffer = '';
$this->tokens = array();
$this->seenTag = false;
$this->lineStart = 0;
$this->line = 0;
$this->otag = '{{';
$this->ctag = '}}';
$this->pragmas = array();
}
/**
@@ -210,62 +201,15 @@ class Mustache_Tokenizer
private function flushBuffer()
{
if (!empty($this->buffer)) {
$this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => $this->buffer);
$this->tokens[] = array(
self::TYPE => self::T_TEXT,
self::LINE => $this->line,
self::VALUE => $this->buffer
);
$this->buffer = '';
}
}
/**
* Test whether the current line is entirely made up of whitespace.
*
* @return boolean True if the current line is all whitespace
*/
private function lineIsWhitespace()
{
$tokensCount = count($this->tokens);
for ($j = $this->lineStart; $j < $tokensCount; $j++) {
$token = $this->tokens[$j];
if (isset(self::$tagTypes[$token[self::TYPE]])) {
if (isset(self::$interpolatedTags[$token[self::TYPE]])) {
return false;
}
} elseif ($token[self::TYPE] == self::T_TEXT) {
if (preg_match('/\S/', $token[self::VALUE])) {
return false;
}
}
}
return true;
}
/**
* Filter out whitespace-only lines and store indent levels for partials.
*
* @param bool $noNewLine Suppress the newline? (default: false)
*/
private function filterLine($noNewLine = false)
{
$this->flushBuffer();
if ($this->seenTag && $this->lineIsWhitespace()) {
$tokensCount = count($this->tokens);
for ($j = $this->lineStart; $j < $tokensCount; $j++) {
if ($this->tokens[$j][self::TYPE] == self::T_TEXT) {
if (isset($this->tokens[$j+1]) && $this->tokens[$j+1][self::TYPE] == self::T_PARTIAL) {
$this->tokens[$j+1][self::INDENT] = $this->tokens[$j][self::VALUE];
}
$this->tokens[$j] = null;
}
}
} elseif (!$noNewLine) {
$this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => "\n");
}
$this->seenTag = false;
$this->lineStart = count($this->tokens);
}
/**
* Change the current Mustache delimiters. Set new `otag` and `ctag` values.
*
@@ -284,11 +228,19 @@ class Mustache_Tokenizer
$this->otag = $otag;
$this->ctag = $ctag;
$this->tokens[] = array(
self::TYPE => self::T_DELIM_CHANGE,
self::LINE => $this->line,
);
return $closeIndex + strlen($close) - 1;
}
/**
* Add pragma tag this template's list of pragmas.
* Add pragma token.
*
* Pragmas are hoisted to the front of the template, so all pragma tokens
* will appear at the front of the token list.
*
* @param string $text
* @param int $index
@@ -297,8 +249,15 @@ class Mustache_Tokenizer
*/
private function addPragma($text, $index)
{
$end = strpos($text, $this->ctag, $index);
$this->pragmas[] = trim(substr($text, $index + 2, $end - $index - 2));
$end = strpos($text, $this->ctag, $index);
$pragma = trim(substr($text, $index + 2, $end - $index - 2));
// Pragmas are hoisted to the front of the template.
array_unshift($this->tokens, array(
self::TYPE => self::T_PRAGMA,
self::NAME => $pragma,
self::LINE => 0,
));
return $end + strlen($this->ctag) - 1;
}
test/Mustache/Test/CompilerTest.php
+2 -4
View File
@@ -105,8 +105,7 @@ class Mustache_Test_CompilerTest extends PHPUnit_Framework_TestCase
array(
'',
array(
$this->createTextToken('foo'),
$this->createTextToken("\n"),
$this->createTextToken("foo\n"),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::NAME => 'name',
@@ -123,8 +122,7 @@ class Mustache_Test_CompilerTest extends PHPUnit_Framework_TestCase
'UTF-8',
array(
"\nclass Monkey extends Mustache_Template",
'$buffer .= $indent . \'foo\'',
'$buffer .= "\n"',
"\$buffer .= \$indent . 'foo\n';",
'$value = $this->resolveValue($context->find(\'name\'), $context, $indent);',
'$buffer .= htmlspecialchars($value, '.ENT_COMPAT.', \'UTF-8\');',
'$value = $this->resolveValue($context->last(), $context, $indent);',
test/Mustache/Test/ParserTest.php
+22 -2
View File
@@ -35,21 +35,25 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array(
array(array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'text'
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'text',
)),
array(array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'text'
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'text',
)),
),
array(
array(array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::NAME => 'name'
)),
array(array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::NAME => 'name'
)),
),
@@ -58,46 +62,55 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array(
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'foo'
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123,
Mustache_Tokenizer::NAME => 'parent'
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::NAME => 'name'
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 456,
Mustache_Tokenizer::NAME => 'parent'
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'bar'
),
),
array(
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'foo'
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123,
Mustache_Tokenizer::END => 456,
Mustache_Tokenizer::NODES => array(
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::NAME => 'name'
),
),
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'bar'
),
),
@@ -125,6 +138,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,
Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123,
),
),
@@ -136,6 +150,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123,
),
),
@@ -147,6 +162,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,
Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123,
),
),
@@ -158,21 +174,25 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,
Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123,
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,
Mustache_Tokenizer::NAME => 'child',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123,
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,
Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123,
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,
Mustache_Tokenizer::NAME => 'child',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123,
),
),
test/Mustache/Test/TokenizerTest.php
+21 -1
View File
@@ -33,6 +33,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
array(
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'text',
),
),
@@ -44,6 +45,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
array(
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'text',
),
),
@@ -58,6 +60,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'name',
Mustache_Tokenizer::OTAG => '{{',
Mustache_Tokenizer::CTAG => '}}',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 10,
)
)
@@ -69,6 +72,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
array(
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => '{{ name }}',
),
),
@@ -83,6 +87,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'name',
Mustache_Tokenizer::OTAG => '<<<',
Mustache_Tokenizer::CTAG => '>>>',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 12,
)
)
@@ -97,10 +102,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'a',
Mustache_Tokenizer::OTAG => '{{',
Mustache_Tokenizer::CTAG => '}}',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 8,
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => "\n",
),
array(
@@ -108,14 +115,24 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'b',
Mustache_Tokenizer::OTAG => '{{',
Mustache_Tokenizer::CTAG => '}}',
Mustache_Tokenizer::LINE => 1,
Mustache_Tokenizer::INDEX => 18,
),
null,
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 1,
Mustache_Tokenizer::VALUE => " \n",
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_DELIM_CHANGE,
Mustache_Tokenizer::LINE => 2,
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::NAME => 'c',
Mustache_Tokenizer::OTAG => '|',
Mustache_Tokenizer::CTAG => '|',
Mustache_Tokenizer::LINE => 2,
Mustache_Tokenizer::INDEX => 37,
),
array(
@@ -123,10 +140,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'b',
Mustache_Tokenizer::OTAG => '|',
Mustache_Tokenizer::CTAG => '|',
Mustache_Tokenizer::LINE => 2,
Mustache_Tokenizer::INDEX => 37,
),
array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 2,
Mustache_Tokenizer::VALUE => "\n",
),
array(
@@ -134,6 +153,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'd',
Mustache_Tokenizer::OTAG => '|',
Mustache_Tokenizer::CTAG => '|',
Mustache_Tokenizer::LINE => 3,
Mustache_Tokenizer::INDEX => 51,
),