From e00bebd1414f0ce911016331e8496cf65f6ffaa4 Mon Sep 17 00:00:00 2001
From: Justin Hileman <justin@justinhileman.info>
Date: Thu, 20 Feb 2014 09:07:43 -0800
Subject: [PATCH] Override encoding to improve tokenizer performance

Speeds up tokenizing by ~15% for most users, and by 50% for users with mbstring.func_overload set!

See #144
---
 src/Mustache/Tokenizer.php | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/Mustache/Tokenizer.php b/src/Mustache/Tokenizer.php
index 1a765da..de33605 100644
--- a/src/Mustache/Tokenizer.php
+++ b/src/Mustache/Tokenizer.php
@@ -92,6 +92,14 @@ class Mustache_Tokenizer
      */
     public function scan($text, $delimiters = null)
     {
+        // Setting mbstring.func_overload makes things *really* slow.
+        // Let's do everyone a favor and scan this string as ASCII instead.
+        $encoding = null;
+        if (function_exists('mb_internal_encoding') && ini_get('mbstring.func_overload') & 2) {
+            $encoding = mb_internal_encoding();
+            mb_internal_encoding('ASCII');
+        }
+
         $this->reset();
 
         if ($delimiters = trim($delimiters)) {
@@ -107,7 +115,7 @@ class Mustache_Tokenizer
                         $this->flushBuffer();
                         $this->state = self::IN_TAG_TYPE;
                     } else {
-                        $char = substr($text, $i, 1);
+                        $char = $text[$i];
                         $this->buffer .= $char;
                         if ($char == "\n") {
                             $this->flushBuffer();
@@ -118,7 +126,7 @@ class Mustache_Tokenizer
 
                 case self::IN_TAG_TYPE:
                     $i += $this->otagLen - 1;
-                    $char = substr($text, $i + 1, 1);
+                    $char = $text[$i + 1];
                     if (isset(self::$tagTypes[$char])) {
                         $tag = $char;
                         $this->tagType = $tag;
@@ -168,7 +176,7 @@ class Mustache_Tokenizer
                             }
                         }
                     } else {
-                        $this->buffer .= substr($text, $i, 1);
+                        $this->buffer .= $text[$i];
                     }
                     break;
             }
@@ -176,6 +184,11 @@ class Mustache_Tokenizer
 
         $this->flushBuffer();
 
+        // Restore the user's encoding...
+        if ($encoding) {
+            mb_internal_encoding($encoding);
+        }
+
         return $this->tokens;
     }