diff --git a/src/Mustache/Compiler.php b/src/Mustache/Compiler.php index aa8978e..437ea2c 100644 --- a/src/Mustache/Compiler.php +++ b/src/Mustache/Compiler.php @@ -373,13 +373,11 @@ class Mustache_Compiler */ private function text($text, $level) { - if ($text === "\n") { - $this->indentNextLine = true; + $indentNextLine = (substr($text, -1) === "\n"); + $code = sprintf($this->prepare(self::TEXT, $level), $this->flushIndent(), var_export($text, true)); + $this->indentNextLine = $indentNextLine; - return $this->prepare(self::LINE, $level); - } else { - return sprintf($this->prepare(self::TEXT, $level), $this->flushIndent(), var_export($text, true)); - } + return $code; } /** @@ -467,12 +465,12 @@ class Mustache_Compiler */ private function flushIndent() { - if ($this->indentNextLine) { - $this->indentNextLine = false; - - return self::LINE_INDENT; - } else { + if (!$this->indentNextLine) { return ''; } + + $this->indentNextLine = false; + + return self::LINE_INDENT; } } diff --git a/src/Mustache/Parser.php b/src/Mustache/Parser.php index f983ad8..697ce7f 100644 --- a/src/Mustache/Parser.php +++ b/src/Mustache/Parser.php @@ -16,6 +16,8 @@ */ class Mustache_Parser { + private $lineNum; + private $lineTokens; /** * Process an array of Mustache tokens and convert them into a parse tree. @@ -26,6 +28,9 @@ class Mustache_Parser */ public function parse(array $tokens = array()) { + $this->lineNum = -1; + $this->lineTokens = 0; + return $this->buildTree($tokens); } @@ -46,38 +51,62 @@ class Mustache_Parser while (!empty($tokens)) { $token = array_shift($tokens); - if ($token === null) { - continue; + if ($token[Mustache_Tokenizer::LINE] === $this->lineNum) { + $this->lineTokens++; } else { - switch ($token[Mustache_Tokenizer::TYPE]) { - case Mustache_Tokenizer::T_SECTION: - case Mustache_Tokenizer::T_INVERTED: - $nodes[] = $this->buildTree($tokens, $token); - break; - - case Mustache_Tokenizer::T_END_SECTION: - if (!isset($parent)) { - $msg = sprintf('Unexpected closing tag: /%s', $token[Mustache_Tokenizer::NAME]); - throw new Mustache_Exception_SyntaxException($msg, $token); - } - - if ($token[Mustache_Tokenizer::NAME] !== $parent[Mustache_Tokenizer::NAME]) { - $msg = sprintf('Nesting error: %s vs. %s', $parent[Mustache_Tokenizer::NAME], $token[Mustache_Tokenizer::NAME]); - throw new Mustache_Exception_SyntaxException($msg, $token); - } - - $parent[Mustache_Tokenizer::END] = $token[Mustache_Tokenizer::INDEX]; - $parent[Mustache_Tokenizer::NODES] = $nodes; - - return $parent; - break; - - default: - $nodes[] = $token; - break; - } + $this->lineNum = $token[Mustache_Tokenizer::LINE]; + $this->lineTokens = 0; } - }; + + switch ($token[Mustache_Tokenizer::TYPE]) { + case Mustache_Tokenizer::T_DELIM_CHANGE: + $this->clearStandaloneLines($nodes, $tokens); + break; + + case Mustache_Tokenizer::T_SECTION: + case Mustache_Tokenizer::T_INVERTED: + $this->clearStandaloneLines($nodes, $tokens); + $nodes[] = $this->buildTree($tokens, $token); + break; + + case Mustache_Tokenizer::T_END_SECTION: + if (!isset($parent)) { + $msg = sprintf('Unexpected closing tag: /%s', $token[Mustache_Tokenizer::NAME]); + throw new Mustache_Exception_SyntaxException($msg, $token); + } + + if ($token[Mustache_Tokenizer::NAME] !== $parent[Mustache_Tokenizer::NAME]) { + $msg = sprintf('Nesting error: %s vs. %s', $parent[Mustache_Tokenizer::NAME], $token[Mustache_Tokenizer::NAME]); + throw new Mustache_Exception_SyntaxException($msg, $token); + } + + $this->clearStandaloneLines($nodes, $tokens); + $parent[Mustache_Tokenizer::END] = $token[Mustache_Tokenizer::INDEX]; + $parent[Mustache_Tokenizer::NODES] = $nodes; + + return $parent; + break; + + case Mustache_Tokenizer::T_PARTIAL: + case Mustache_Tokenizer::T_PARTIAL_2: + // store the whitespace prefix for laters! + if ($indent = $this->clearStandaloneLines($nodes, $tokens)) { + $token[Mustache_Tokenizer::INDENT] = $indent[Mustache_Tokenizer::VALUE]; + } + $nodes[] = $token; + break; + + case Mustache_Tokenizer::T_PRAGMA: + case Mustache_Tokenizer::T_COMMENT: + $this->clearStandaloneLines($nodes, $tokens); + $nodes[] = $token; + break; + + default: + $nodes[] = $token; + break; + } + } if (isset($parent)) { $msg = sprintf('Missing closing tag: %s', $parent[Mustache_Tokenizer::NAME]); @@ -86,4 +115,80 @@ class Mustache_Parser return $nodes; } + + /** + * Clear standalone line tokens. + * + * Returns a whitespace token for indenting partials, if applicable. + * + * @param array $nodes Parsed nodes. + * @param array $tokens Tokens to be parsed. + * + * @return array Resulting indent token, if any. + */ + private function clearStandaloneLines(array &$nodes, array &$tokens) + { + if ($this->lineTokens > 1) { + // this is the third or later node on this line, so it can't be standalone + return; + } + + $prev = null; + if ($this->lineTokens === 1) { + // this is the second node on this line, so it can't be standalone + // unless the previous node is whitespace. + if ($prev = end($nodes)) { + if (!$this->tokenIsWhitespace($prev)) { + return; + } + } + } + + $next = null; + if ($next = reset($tokens)) { + // If we're on a new line, bail. + if ($next[Mustache_Tokenizer::LINE] !== $this->lineNum) { + return; + } + + // If the next token isn't whitespace, bail. + if (!$this->tokenIsWhitespace($next)) { + return; + } + + if (count($tokens) !== 1) { + // Unless it's the last token in the template, the next token + // must end in newline for this to be standalone. + if (substr($next[Mustache_Tokenizer::VALUE], -1) !== "\n") { + return; + } + } + + // Discard the whitespace suffix + array_shift($tokens); + } + + if ($prev) { + // Return the whitespace prefix, if any + return array_pop($nodes); + } + } + + /** + * Check whether token is a whitespace token. + * + * True if token type is T_TEXT and value is all whitespace characters. + * + * @param array $token + * + * @return boolean True if token is a whitespace token + */ + private function tokenIsWhitespace(array $token) + { + if ($token[Mustache_Tokenizer::TYPE] == Mustache_Tokenizer::T_TEXT) { + return preg_match('/^\s*$/', $token[Mustache_Tokenizer::VALUE]); + } + + return false; + } } diff --git a/src/Mustache/Tokenizer.php b/src/Mustache/Tokenizer.php index 99e7056..e065622 100644 --- a/src/Mustache/Tokenizer.php +++ b/src/Mustache/Tokenizer.php @@ -63,20 +63,20 @@ class Mustache_Tokenizer const NAME = 'name'; const OTAG = 'otag'; const CTAG = 'ctag'; + const LINE = 'line'; const INDEX = 'index'; const END = 'end'; const INDENT = 'indent'; const NODES = 'nodes'; const VALUE = 'value'; - private $pragmas; private $state; private $tagType; private $tag; private $buffer; private $tokens; private $seenTag; - private $lineStart; + private $line; private $otag; private $ctag; @@ -108,16 +108,15 @@ class Mustache_Tokenizer $this->state = self::IN_TAG_TYPE; } else { $char = substr($text, $i, 1); + $this->buffer .= $char; if ($char == "\n") { - $this->filterLine(); - } else { - $this->buffer .= $char; + $this->flushBuffer(); + $this->line++; } } break; case self::IN_TAG_TYPE: - $i += strlen($this->otag) - 1; $char = substr($text, $i + 1, 1); if (isset(self::$tagTypes[$char])) { @@ -150,6 +149,7 @@ class Mustache_Tokenizer self::NAME => trim($this->buffer), self::OTAG => $this->otag, self::CTAG => $this->ctag, + self::LINE => $this->line, self::INDEX => ($this->tagType == self::T_END_SECTION) ? $this->seenTag - strlen($this->otag) : $i + strlen($this->ctag) ); @@ -174,15 +174,7 @@ class Mustache_Tokenizer } } - $this->filterLine(true); - - // Pragmas are hoisted to the front of the template. - foreach ($this->pragmas as $pragma) { - array_unshift($this->tokens, array( - self::TYPE => self::T_PRAGMA, - self::NAME => $pragma, - )); - } + $this->flushBuffer(); return $this->tokens; } @@ -198,10 +190,9 @@ class Mustache_Tokenizer $this->buffer = ''; $this->tokens = array(); $this->seenTag = false; - $this->lineStart = 0; + $this->line = 0; $this->otag = '{{'; $this->ctag = '}}'; - $this->pragmas = array(); } /** @@ -210,62 +201,15 @@ class Mustache_Tokenizer private function flushBuffer() { if (!empty($this->buffer)) { - $this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => $this->buffer); + $this->tokens[] = array( + self::TYPE => self::T_TEXT, + self::LINE => $this->line, + self::VALUE => $this->buffer + ); $this->buffer = ''; } } - /** - * Test whether the current line is entirely made up of whitespace. - * - * @return boolean True if the current line is all whitespace - */ - private function lineIsWhitespace() - { - $tokensCount = count($this->tokens); - for ($j = $this->lineStart; $j < $tokensCount; $j++) { - $token = $this->tokens[$j]; - if (isset(self::$tagTypes[$token[self::TYPE]])) { - if (isset(self::$interpolatedTags[$token[self::TYPE]])) { - return false; - } - } elseif ($token[self::TYPE] == self::T_TEXT) { - if (preg_match('/\S/', $token[self::VALUE])) { - return false; - } - } - } - - return true; - } - - /** - * Filter out whitespace-only lines and store indent levels for partials. - * - * @param bool $noNewLine Suppress the newline? (default: false) - */ - private function filterLine($noNewLine = false) - { - $this->flushBuffer(); - if ($this->seenTag && $this->lineIsWhitespace()) { - $tokensCount = count($this->tokens); - for ($j = $this->lineStart; $j < $tokensCount; $j++) { - if ($this->tokens[$j][self::TYPE] == self::T_TEXT) { - if (isset($this->tokens[$j+1]) && $this->tokens[$j+1][self::TYPE] == self::T_PARTIAL) { - $this->tokens[$j+1][self::INDENT] = $this->tokens[$j][self::VALUE]; - } - - $this->tokens[$j] = null; - } - } - } elseif (!$noNewLine) { - $this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => "\n"); - } - - $this->seenTag = false; - $this->lineStart = count($this->tokens); - } - /** * Change the current Mustache delimiters. Set new `otag` and `ctag` values. * @@ -284,11 +228,19 @@ class Mustache_Tokenizer $this->otag = $otag; $this->ctag = $ctag; + $this->tokens[] = array( + self::TYPE => self::T_DELIM_CHANGE, + self::LINE => $this->line, + ); + return $closeIndex + strlen($close) - 1; } /** - * Add pragma tag this template's list of pragmas. + * Add pragma token. + * + * Pragmas are hoisted to the front of the template, so all pragma tokens + * will appear at the front of the token list. * * @param string $text * @param int $index @@ -297,8 +249,15 @@ class Mustache_Tokenizer */ private function addPragma($text, $index) { - $end = strpos($text, $this->ctag, $index); - $this->pragmas[] = trim(substr($text, $index + 2, $end - $index - 2)); + $end = strpos($text, $this->ctag, $index); + $pragma = trim(substr($text, $index + 2, $end - $index - 2)); + + // Pragmas are hoisted to the front of the template. + array_unshift($this->tokens, array( + self::TYPE => self::T_PRAGMA, + self::NAME => $pragma, + self::LINE => 0, + )); return $end + strlen($this->ctag) - 1; } diff --git a/test/Mustache/Test/CompilerTest.php b/test/Mustache/Test/CompilerTest.php index e429e73..a5a2bdf 100644 --- a/test/Mustache/Test/CompilerTest.php +++ b/test/Mustache/Test/CompilerTest.php @@ -105,8 +105,7 @@ class Mustache_Test_CompilerTest extends PHPUnit_Framework_TestCase array( '', array( - $this->createTextToken('foo'), - $this->createTextToken("\n"), + $this->createTextToken("foo\n"), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, Mustache_Tokenizer::NAME => 'name', @@ -123,8 +122,7 @@ class Mustache_Test_CompilerTest extends PHPUnit_Framework_TestCase 'UTF-8', array( "\nclass Monkey extends Mustache_Template", - '$buffer .= $indent . \'foo\'', - '$buffer .= "\n"', + "\$buffer .= \$indent . 'foo\n';", '$value = $this->resolveValue($context->find(\'name\'), $context, $indent);', '$buffer .= htmlspecialchars($value, '.ENT_COMPAT.', \'UTF-8\');', '$value = $this->resolveValue($context->last(), $context, $indent);', diff --git a/test/Mustache/Test/ParserTest.php b/test/Mustache/Test/ParserTest.php index a99b848..ea11510 100644 --- a/test/Mustache/Test/ParserTest.php +++ b/test/Mustache/Test/ParserTest.php @@ -35,21 +35,25 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase array( array(array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, - Mustache_Tokenizer::VALUE => 'text' + Mustache_Tokenizer::LINE => 0, + Mustache_Tokenizer::VALUE => 'text', )), array(array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, - Mustache_Tokenizer::VALUE => 'text' + Mustache_Tokenizer::LINE => 0, + Mustache_Tokenizer::VALUE => 'text', )), ), array( array(array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::NAME => 'name' )), array(array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::NAME => 'name' )), ), @@ -58,46 +62,55 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase array( array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::VALUE => 'foo' ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::NAME => 'parent' ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::NAME => 'name' ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 456, Mustache_Tokenizer::NAME => 'parent' ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::VALUE => 'bar' ), ), array( array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::VALUE => 'foo' ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED, Mustache_Tokenizer::NAME => 'parent', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::END => 456, Mustache_Tokenizer::NODES => array( array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::NAME => 'name' ), ), ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::VALUE => 'bar' ), ), @@ -125,6 +138,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION, Mustache_Tokenizer::NAME => 'parent', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 123, ), ), @@ -136,6 +150,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED, Mustache_Tokenizer::NAME => 'parent', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 123, ), ), @@ -147,6 +162,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION, Mustache_Tokenizer::NAME => 'parent', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 123, ), ), @@ -158,21 +174,25 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION, Mustache_Tokenizer::NAME => 'parent', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 123, ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION, Mustache_Tokenizer::NAME => 'child', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 123, ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION, Mustache_Tokenizer::NAME => 'parent', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 123, ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION, Mustache_Tokenizer::NAME => 'child', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 123, ), ), diff --git a/test/Mustache/Test/TokenizerTest.php b/test/Mustache/Test/TokenizerTest.php index 3f6d76b..fa98b84 100644 --- a/test/Mustache/Test/TokenizerTest.php +++ b/test/Mustache/Test/TokenizerTest.php @@ -33,6 +33,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase array( array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::VALUE => 'text', ), ), @@ -44,6 +45,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase array( array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::VALUE => 'text', ), ), @@ -58,6 +60,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase Mustache_Tokenizer::NAME => 'name', Mustache_Tokenizer::OTAG => '{{', Mustache_Tokenizer::CTAG => '}}', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 10, ) ) @@ -69,6 +72,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase array( array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::VALUE => '{{ name }}', ), ), @@ -83,6 +87,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase Mustache_Tokenizer::NAME => 'name', Mustache_Tokenizer::OTAG => '<<<', Mustache_Tokenizer::CTAG => '>>>', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 12, ) ) @@ -97,10 +102,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase Mustache_Tokenizer::NAME => 'a', Mustache_Tokenizer::OTAG => '{{', Mustache_Tokenizer::CTAG => '}}', + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::INDEX => 8, ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 0, Mustache_Tokenizer::VALUE => "\n", ), array( @@ -108,14 +115,24 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase Mustache_Tokenizer::NAME => 'b', Mustache_Tokenizer::OTAG => '{{', Mustache_Tokenizer::CTAG => '}}', + Mustache_Tokenizer::LINE => 1, Mustache_Tokenizer::INDEX => 18, ), - null, + array( + Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 1, + Mustache_Tokenizer::VALUE => " \n", + ), + array( + Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_DELIM_CHANGE, + Mustache_Tokenizer::LINE => 2, + ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, Mustache_Tokenizer::NAME => 'c', Mustache_Tokenizer::OTAG => '|', Mustache_Tokenizer::CTAG => '|', + Mustache_Tokenizer::LINE => 2, Mustache_Tokenizer::INDEX => 37, ), array( @@ -123,10 +140,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase Mustache_Tokenizer::NAME => 'b', Mustache_Tokenizer::OTAG => '|', Mustache_Tokenizer::CTAG => '|', + Mustache_Tokenizer::LINE => 2, Mustache_Tokenizer::INDEX => 37, ), array( Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, + Mustache_Tokenizer::LINE => 2, Mustache_Tokenizer::VALUE => "\n", ), array( @@ -134,6 +153,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase Mustache_Tokenizer::NAME => 'd', Mustache_Tokenizer::OTAG => '|', Mustache_Tokenizer::CTAG => '|', + Mustache_Tokenizer::LINE => 3, Mustache_Tokenizer::INDEX => 51, ),