Track line number in Tokenizer

(needed for standalone line filtering via Parser)

src/Mustache/Tokenizer.php

@@ -63,6 +63,7 @@ class Mustache_Tokenizer
     const NAME   = 'name';
     const OTAG   = 'otag';
     const CTAG   = 'ctag';
+    const LINE   = 'line';
     const INDEX  = 'index';
     const END    = 'end';
     const INDENT = 'indent';
@@ -76,7 +77,7 @@ class Mustache_Tokenizer
     private $buffer;
     private $tokens;
     private $seenTag;
-    private $lineStart;
+    private $line;
     private $otag;
     private $ctag;
 
@@ -111,6 +112,7 @@ class Mustache_Tokenizer
                         $this->buffer .= $char;
                         if ($char == "\n") {
                             $this->flushBuffer();
+                            $this->line++;
                         }
                     }
                     break;
@@ -148,6 +150,7 @@ class Mustache_Tokenizer
                             self::NAME  => trim($this->buffer),
                             self::OTAG  => $this->otag,
                             self::CTAG  => $this->ctag,
+                            self::LINE  => $this->line,
                             self::INDEX => ($this->tagType == self::T_END_SECTION) ? $this->seenTag - strlen($this->otag) : $i + strlen($this->ctag)
                         );
 
@@ -196,6 +199,7 @@ class Mustache_Tokenizer
         $this->buffer    = '';
         $this->tokens    = array();
         $this->seenTag   = false;
+        $this->line      = 0;
         $this->otag      = '{{';
         $this->ctag      = '}}';
         $this->pragmas   = array();
@@ -207,7 +211,11 @@ class Mustache_Tokenizer
     private function flushBuffer()
     {
         if (!empty($this->buffer)) {
-            $this->tokens[] = array(self::TYPE  => self::T_TEXT, self::VALUE => $this->buffer);
+            $this->tokens[] = array(
+                self::TYPE  => self::T_TEXT,
+                self::LINE  => $this->line,
+                self::VALUE => $this->buffer
+            );
             $this->buffer   = '';
         }
     }

test/Mustache/Test/ParserTest.php

@@ -35,21 +35,25 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
             array(
                 array(array(
                     Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
-                    Mustache_Tokenizer::VALUE => 'text'
+                    Mustache_Tokenizer::LINE  => 0,
+                    Mustache_Tokenizer::VALUE => 'text',
                 )),
                 array(array(
                     Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
-                    Mustache_Tokenizer::VALUE => 'text'
+                    Mustache_Tokenizer::LINE  => 0,
+                    Mustache_Tokenizer::VALUE => 'text',
                 )),
             ),
 
             array(
                 array(array(
                     Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
+                    Mustache_Tokenizer::LINE => 0,
                     Mustache_Tokenizer::NAME => 'name'
                 )),
                 array(array(
                     Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
+                    Mustache_Tokenizer::LINE => 0,
                     Mustache_Tokenizer::NAME => 'name'
                 )),
             ),
@@ -58,46 +62,55 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
                 array(
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::VALUE => 'foo'
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_INVERTED,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 123,
                         Mustache_Tokenizer::NAME  => 'parent'
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_ESCAPED,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::NAME  => 'name'
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 456,
                         Mustache_Tokenizer::NAME  => 'parent'
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::VALUE => 'bar'
                     ),
                 ),
                 array(
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::VALUE => 'foo'
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_INVERTED,
                         Mustache_Tokenizer::NAME  => 'parent',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 123,
                         Mustache_Tokenizer::END   => 456,
                         Mustache_Tokenizer::NODES => array(
                             array(
                                 Mustache_Tokenizer::TYPE => Mustache_Tokenizer::T_ESCAPED,
+                                Mustache_Tokenizer::LINE => 0,
                                 Mustache_Tokenizer::NAME => 'name'
                             ),
                         ),
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::VALUE => 'bar'
                     ),
                 ),
@@ -125,6 +138,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_SECTION,
                         Mustache_Tokenizer::NAME  => 'parent',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 123,
                     ),
                 ),
@@ -136,6 +150,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_INVERTED,
                         Mustache_Tokenizer::NAME  => 'parent',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 123,
                     ),
                 ),
@@ -147,6 +162,7 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,
                         Mustache_Tokenizer::NAME  => 'parent',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 123,
                     ),
                 ),
@@ -158,21 +174,25 @@ class Mustache_Test_ParserTest extends PHPUnit_Framework_TestCase
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_SECTION,
                         Mustache_Tokenizer::NAME  => 'parent',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 123,
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_SECTION,
                         Mustache_Tokenizer::NAME  => 'child',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 123,
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,
                         Mustache_Tokenizer::NAME  => 'parent',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 123,
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_END_SECTION,
                         Mustache_Tokenizer::NAME  => 'child',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 123,
                     ),
                 ),

test/Mustache/Test/TokenizerTest.php

@@ -33,6 +33,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                 array(
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::VALUE => 'text',
                     ),
                 ),
@@ -44,6 +45,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                 array(
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::VALUE => 'text',
                     ),
                 ),
@@ -58,6 +60,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                         Mustache_Tokenizer::NAME  => 'name',
                         Mustache_Tokenizer::OTAG  => '{{',
                         Mustache_Tokenizer::CTAG  => '}}',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 10,
                     )
                 )
@@ -69,6 +72,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                 array(
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::VALUE => '{{ name }}',
                     ),
                 ),
@@ -83,6 +87,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                         Mustache_Tokenizer::NAME  => 'name',
                         Mustache_Tokenizer::OTAG  => '<<<',
                         Mustache_Tokenizer::CTAG  => '>>>',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 12,
                     )
                 )
@@ -97,10 +102,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                         Mustache_Tokenizer::NAME  => 'a',
                         Mustache_Tokenizer::OTAG  => '{{',
                         Mustache_Tokenizer::CTAG  => '}}',
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::INDEX => 8,
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+                        Mustache_Tokenizer::LINE  => 0,
                         Mustache_Tokenizer::VALUE => "\n",
                     ),
                     array(
@@ -108,6 +115,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                         Mustache_Tokenizer::NAME  => 'b',
                         Mustache_Tokenizer::OTAG  => '{{',
                         Mustache_Tokenizer::CTAG  => '}}',
+                        Mustache_Tokenizer::LINE  => 1,
                         Mustache_Tokenizer::INDEX => 18,
                     ),
                     array(
@@ -120,6 +128,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                         Mustache_Tokenizer::NAME  => 'c',
                         Mustache_Tokenizer::OTAG  => '|',
                         Mustache_Tokenizer::CTAG  => '|',
+                        Mustache_Tokenizer::LINE  => 2,
                         Mustache_Tokenizer::INDEX => 37,
                     ),
                     array(
@@ -127,10 +136,12 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                         Mustache_Tokenizer::NAME  => 'b',
                         Mustache_Tokenizer::OTAG  => '|',
                         Mustache_Tokenizer::CTAG  => '|',
+                        Mustache_Tokenizer::LINE  => 2,
                         Mustache_Tokenizer::INDEX => 37,
                     ),
                     array(
                         Mustache_Tokenizer::TYPE  => Mustache_Tokenizer::T_TEXT,
+                        Mustache_Tokenizer::LINE  => 2,
                         Mustache_Tokenizer::VALUE => "\n",
                     ),
                     array(
@@ -138,6 +149,7 @@ class Mustache_Test_TokenizerTest extends PHPUnit_Framework_TestCase
                         Mustache_Tokenizer::NAME  => 'd',
                         Mustache_Tokenizer::OTAG  => '|',
                         Mustache_Tokenizer::CTAG  => '|',
+                        Mustache_Tokenizer::LINE  => 3,
                         Mustache_Tokenizer::INDEX => 51,
                     ),