Merge pull request #38 from egulias/improved-utf8

#30 - Improved utf8
egulias · Nov 29, 2014 · 7a64ea1 · 7a64ea1
2 parents 3f623e9 + 9103f4f
commit 7a64ea1
Show file tree

Hide file tree

Showing 6 changed files with 163 additions and 70 deletions.
diff --git a/composer.json b/composer.json
@@ -15,7 +15,7 @@
   },
   "require":      {
     "php": ">= 5.3.3",
-    "doctrine/lexer": "~1.0"
+    "doctrine/lexer": "dev-master"
   },
   "require-dev" :   {
     "satooshi/php-coveralls": "dev-master"

diff --git a/composer.lock b/composer.lock
diff --git a/src/Egulias/EmailValidator/EmailLexer.php b/src/Egulias/EmailValidator/EmailLexer.php
@@ -143,7 +143,6 @@ protected function getCatchablePatterns()
             '\r\n',
             '::',
             '\s+?',
-            '[\x10-\x1F]+',
             '.',
             );
     }
@@ -155,7 +154,7 @@ protected function getCatchablePatterns()
      */
     protected function getNonCatchablePatterns()
     {
-        return array('[\x7f-\xff]+');
+        return array('[\xA0-\xff]+');
     }
 
     /**
@@ -167,25 +166,39 @@ protected function getNonCatchablePatterns()
      */
     protected function getType(&$value)
     {
-
         if ($this->isNullType($value)) {
             return self::C_NUL;
         }
 
-        if (isset($this->charValue[$value])) {
+        if ($this->isValid($value)) {
             return $this->charValue[$value];
         }
 
-        if ($this->isInvalid($value)) {
+        if ($this->isUTF8Invalid($value)) {
+            $this->hasInvalidTokens = true;
+            return self::INVALID;
+        }
+
+        if ($this->isASCIIInvalid($value)) {
             $this->hasInvalidTokens = true;
             return self::INVALID;
         }
 
         return  self::GENERIC;
     }
 
+    protected function isValid($value)
+    {
+        if (isset($this->charValue[$value])) {
+            return true;
+        }
+
+        return false;
+    }
+
     /**
-     * @param string $value
+     * @param $value
+     * @return bool
      */
     protected function isNullType($value)
     {
@@ -197,18 +210,33 @@ protected function isNullType($value)
     }
 
     /**
-     * @param string $value
+     * @param $value
+     * @return bool
      */
-    protected function isInvalid($value)
+    protected function isASCIIInvalid($value)
     {
-        if (preg_match('/[\x10-\x1F]+/', $value)) {
+        if (isset($this->invalidASCII[ord($value)])) {
             return true;
         }
 
-        if (isset($this->invalidASCII[ord($value)])) {
+        return false;
+    }
+
+    /**
+     * @param $value
+     * @return bool
+     */
+    protected function isUTF8Invalid($value)
+    {
+        if (preg_match('/\p{Cc}+/u', $value)) {
             return true;
         }
 
         return false;
     }
+
+    protected function getModifiers()
+    {
+        return 'iu';
+    }
 }