Framework/Mustache
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Track line number in Tokenizer

Browse Source 
(needed for standalone line filtering via Parser)
This commit is contained in:
Justin Hileman
2013-05-15 13:37:05 -07:00
parent b77d2f308e
commit 32461bfb48
3 changed files with 44 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Mustache/Tokenizer.php
+10 -2
View File
@@ -63,6 +63,7 @@ class Mustache_Tokenizer
const NAME = 'name'; const NAME = 'name';
const OTAG = 'otag'; const OTAG = 'otag';
const CTAG = 'ctag'; const CTAG = 'ctag';
const LINE = 'line';
const INDEX = 'index'; const INDEX = 'index';
const END = 'end'; const END = 'end';
const INDENT = 'indent'; const INDENT = 'indent';
@@ -76,7 +77,7 @@ class Mustache_Tokenizer
private $buffer; private $buffer;
private $tokens; private $tokens;
private $seenTag; private $seenTag;
private $lineStart; private $line;
private $otag; private $otag;
private $ctag; private $ctag;
@@ -111,6 +112,7 @@ class Mustache_Tokenizer
$this->buffer .= $char; $this->buffer .= $char;
if ($char == "\n") { if ($char == "\n") {
$this->flushBuffer(); $this->flushBuffer();
$this->line++;
} }
} }
break; break;
@@ -148,6 +150,7 @@ class Mustache_Tokenizer
self::NAME => trim($this->buffer), self::NAME => trim($this->buffer),
self::OTAG => $this->otag, self::OTAG => $this->otag,
self::CTAG => $this->ctag, self::CTAG => $this->ctag,
self::LINE => $this->line,
self::INDEX => ($this->tagType == self::T_END_SECTION) ? $this->seenTag - strlen($this->otag) : $i + strlen($this->ctag) self::INDEX => ($this->tagType == self::T_END_SECTION) ? $this->seenTag - strlen($this->otag) : $i + strlen($this->ctag)
); );
@@ -196,6 +199,7 @@ class Mustache_Tokenizer
$this->buffer = ''; $this->buffer = '';
$this->tokens = array(); $this->tokens = array();
$this->seenTag = false; $this->seenTag = false;
$this->line = 0;
$this->otag = '{{'; $this->otag = '{{';
$this->ctag = '}}'; $this->ctag = '}}';
$this->pragmas = array(); $this->pragmas = array();
@@ -207,7 +211,11 @@ class Mustache_Tokenizer
private function flushBuffer() private function flushBuffer()
{ {
if (!empty($this->buffer)) { if (!empty($this->buffer)) {
$this->tokens[] = array(self::TYPE => self::T_TEXT, self::VALUE => $this->buffer); $this->tokens[] = array(
self::TYPE => self::T_TEXT,
self::LINE => $this->line,
self::VALUE => $this->buffer
);
$this->buffer = ''; $this->buffer = '';
} }
} }
test/Mustache/Test/ParserTest.php
+22 -2
View File
@@ -35,21 +35,25 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array( array(
array(array( array(array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'text' Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'text',
)), )),
array(array( array(array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::VALUE => 'text' Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'text',
)), )),
), ),
array( array(
array(array( array(array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::NAME => 'name' Mustache_Tokenizer::NAME => 'name'
)), )),
array(array( array(array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::NAME => 'name' Mustache_Tokenizer::NAME => 'name'
)), )),
), ),
@@ -58,46 +62,55 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array( array(
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'foo' Mustache_Tokenizer::VALUE => 'foo'
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
Mustache_Tokenizer::NAME => 'parent' Mustache_Tokenizer::NAME => 'parent'
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::NAME => 'name' Mustache_Tokenizer::NAME => 'name'
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 456, Mustache_Tokenizer::INDEX => 456,
Mustache_Tokenizer::NAME => 'parent' Mustache_Tokenizer::NAME => 'parent'
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'bar' Mustache_Tokenizer::VALUE => 'bar'
), ),
), ),
array( array(
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'foo' Mustache_Tokenizer::VALUE => 'foo'
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::NAME => 'parent', Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
Mustache_Tokenizer::END => 456, Mustache_Tokenizer::END => 456,
Mustache_Tokenizer::NODES => array( Mustache_Tokenizer::NODES => array(
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::NAME => 'name' Mustache_Tokenizer::NAME => 'name'
), ),
), ),
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'bar' Mustache_Tokenizer::VALUE => 'bar'
), ),
), ),
@@ -125,6 +138,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,
Mustache_Tokenizer::NAME => 'parent', Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
), ),
), ),
@@ -136,6 +150,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_INVERTED,
Mustache_Tokenizer::NAME => 'parent', Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
), ),
), ),
@@ -147,6 +162,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,
Mustache_Tokenizer::NAME => 'parent', Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
), ),
), ),
@@ -158,21 +174,25 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,
Mustache_Tokenizer::NAME => 'parent', Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_SECTION,
Mustache_Tokenizer::NAME => 'child', Mustache_Tokenizer::NAME => 'child',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,
Mustache_Tokenizer::NAME => 'parent', Mustache_Tokenizer::NAME => 'parent',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_END_SECTION,
Mustache_Tokenizer::NAME => 'child', Mustache_Tokenizer::NAME => 'child',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 123, Mustache_Tokenizer::INDEX => 123,
), ),
), ),
test/Mustache/Test/TokenizerTest.php
+12
View File
@@ -33,6 +33,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
array( array(
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'text', Mustache_Tokenizer::VALUE => 'text',
), ),
), ),
@@ -44,6 +45,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
array( array(
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => 'text', Mustache_Tokenizer::VALUE => 'text',
), ),
), ),
@@ -58,6 +60,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'name', Mustache_Tokenizer::NAME => 'name',
Mustache_Tokenizer::OTAG => '{{', Mustache_Tokenizer::OTAG => '{{',
Mustache_Tokenizer::CTAG => '}}', Mustache_Tokenizer::CTAG => '}}',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 10, Mustache_Tokenizer::INDEX => 10,
) )
) )
@@ -69,6 +72,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
array( array(
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => '{{ name }}', Mustache_Tokenizer::VALUE => '{{ name }}',
), ),
), ),
@@ -83,6 +87,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'name', Mustache_Tokenizer::NAME => 'name',
Mustache_Tokenizer::OTAG => '<<<', Mustache_Tokenizer::OTAG => '<<<',
Mustache_Tokenizer::CTAG => '>>>', Mustache_Tokenizer::CTAG => '>>>',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 12, Mustache_Tokenizer::INDEX => 12,
) )
) )
@@ -97,10 +102,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'a', Mustache_Tokenizer::NAME => 'a',
Mustache_Tokenizer::OTAG => '{{', Mustache_Tokenizer::OTAG => '{{',
Mustache_Tokenizer::CTAG => '}}', Mustache_Tokenizer::CTAG => '}}',
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::INDEX => 8, Mustache_Tokenizer::INDEX => 8,
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 0,
Mustache_Tokenizer::VALUE => "\n", Mustache_Tokenizer::VALUE => "\n",
), ),
array( array(
@@ -108,6 +115,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'b', Mustache_Tokenizer::NAME => 'b',
Mustache_Tokenizer::OTAG => '{{', Mustache_Tokenizer::OTAG => '{{',
Mustache_Tokenizer::CTAG => '}}', Mustache_Tokenizer::CTAG => '}}',
Mustache_Tokenizer::LINE => 1,
Mustache_Tokenizer::INDEX => 18, Mustache_Tokenizer::INDEX => 18,
), ),
array( array(
@@ -120,6 +128,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'c', Mustache_Tokenizer::NAME => 'c',
Mustache_Tokenizer::OTAG => '|', Mustache_Tokenizer::OTAG => '|',
Mustache_Tokenizer::CTAG => '|', Mustache_Tokenizer::CTAG => '|',
Mustache_Tokenizer::LINE => 2,
Mustache_Tokenizer::INDEX => 37, Mustache_Tokenizer::INDEX => 37,
), ),
array( array(
@@ -127,10 +136,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'b', Mustache_Tokenizer::NAME => 'b',
Mustache_Tokenizer::OTAG => '|', Mustache_Tokenizer::OTAG => '|',
Mustache_Tokenizer::CTAG => '|', Mustache_Tokenizer::CTAG => '|',
Mustache_Tokenizer::LINE => 2,
Mustache_Tokenizer::INDEX => 37, Mustache_Tokenizer::INDEX => 37,
), ),
array( array(
Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT, Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_TEXT,
Mustache_Tokenizer::LINE => 2,
Mustache_Tokenizer::VALUE => "\n", Mustache_Tokenizer::VALUE => "\n",
), ),
array( array(
@@ -138,6 +149,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
Mustache_Tokenizer::NAME => 'd', Mustache_Tokenizer::NAME => 'd',
Mustache_Tokenizer::OTAG => '|', Mustache_Tokenizer::OTAG => '|',
Mustache_Tokenizer::CTAG => '|', Mustache_Tokenizer::CTAG => '|',
Mustache_Tokenizer::LINE => 3,
Mustache_Tokenizer::INDEX => 51, Mustache_Tokenizer::INDEX => 51,
), ),