Merge pull request #45 from twitter/allow_@_in_url_and_add_rtl_class

Allow @ in url, add rtl class, update t.co url lengths

twitter-archive/twitter-text-conformance/pull/50

pom.xml

@@ -63,12 +63,10 @@
     </dependency>
 
     <dependency>
-      <groupId>org.jyaml</groupId>
-      <artifactId>jyaml</artifactId>
-      <version>1.3</version>
-      <scope>test</scope>
+      <groupId>org.yaml</groupId>
+      <artifactId>snakeyaml</artifactId>
+      <version>1.11</version>
     </dependency>
 
-
   </dependencies>
 </project>

src/com/twitter/Autolink.java

@@ -147,7 +147,12 @@ public void linkToHashtag(Entity entity, String text, StringBuilder builder) {
     Map<String, String> attrs = new LinkedHashMap<String, String>();
     attrs.put("href", hashtagUrlBase + hashtag);
     attrs.put("title", "#" + hashtag);
-    attrs.put("class", hashtagClass);
+
+    if (Regex.RTL_CHARACTERS.matcher(text).find()) {
+      attrs.put("class", hashtagClass + " rtl");
+    } else {
+      attrs.put("class", hashtagClass);
+    }
 
     linkToTextWithSymbol(entity, hashChar, hashtag, attrs, builder);
   }

src/com/twitter/Regex.java

@@ -96,7 +96,7 @@ public class Regex {
 
   private static final String URL_VALID_PORT_NUMBER = "[0-9]++";
 
-  private static final String URL_VALID_GENERAL_PATH_CHARS = "[a-z0-9!\\*';:=\\+,.\\$/%#\\[\\]\\-_~\\|&" + LATIN_ACCENTS_CHARS + "]";
+  private static final String URL_VALID_GENERAL_PATH_CHARS = "[a-z0-9!\\*';:=\\+,.\\$/%#\\[\\]\\-_~\\|&@" + LATIN_ACCENTS_CHARS + "]";
   /** Allow URL paths to contain balanced parens
    *  1. Used in Wikipedia URLs like /Primer_(film)
    *  2. Used in IIS sessions like /S(dfd346)/
@@ -115,7 +115,7 @@ public class Regex {
     ")|(?:@" + URL_VALID_GENERAL_PATH_CHARS + "+/)" +
   ")";
 
-  private static final String URL_VALID_URL_QUERY_CHARS = "[a-z0-9!?\\*'\\(\\);:&=\\+\\$/%#\\[\\]\\-_\\.,~\\|]";
+  private static final String URL_VALID_URL_QUERY_CHARS = "[a-z0-9!?\\*'\\(\\);:&=\\+\\$/%#\\[\\]\\-_\\.,~\\|@]";
   private static final String URL_VALID_URL_QUERY_ENDING_CHARS = "[a-z0-9_&=#/]";
   private static final String VALID_URL_PATTERN_STRING =
   "(" +                                                            //  $1 total match
@@ -144,6 +144,7 @@ public class Regex {
   public static final int VALID_HASHTAG_GROUP_HASH = 2;
   public static final int VALID_HASHTAG_GROUP_TAG = 3;
   public static final Pattern INVALID_HASHTAG_MATCH_END = Pattern.compile("^(?:[#＃]|://)");
+  public static final Pattern RTL_CHARACTERS = Pattern.compile("[\u0600-\u06FF\u0750-\u077F\u0590-\u05FF\uFE70-\uFEFF]");
 
   public static final Pattern AT_SIGNS = Pattern.compile("[" + AT_SIGNS_CHARS + "]");
   public static final Pattern VALID_MENTION_OR_LIST = Pattern.compile("([^a-z0-9_!#$%&*" + AT_SIGNS_CHARS + "]|^|RT:?)(" + AT_SIGNS + "+)([a-z0-9_]{1,20})(/[a-z][a-z0-9_\\-]{0,24})?", Pattern.CASE_INSENSITIVE);

src/com/twitter/Validator.java

@@ -8,8 +8,8 @@
 public class Validator {
   public static final int MAX_TWEET_LENGTH = 140;
 
-  protected int shortUrlLength = 20;
-  protected int shortUrlLengthHttps = 21;
+  protected int shortUrlLength = 22;
+  protected int shortUrlLengthHttps = 23;
 
   private Extractor extractor = new Extractor();
 

tests/com/twitter/ConformanceTest.java

@@ -5,10 +5,11 @@
 import java.util.List;
 import java.util.ArrayList;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 
 import junit.framework.TestCase;
-import org.ho.yaml.Yaml;
+import org.yaml.snakeyaml.Yaml;
 
 import com.twitter.Extractor.Entity;
 
@@ -219,7 +220,8 @@ protected void autolink(List<Map> testCases) {
   }
 
   protected List loadConformanceData(File yamlFile, String testType) throws FileNotFoundException {
-    Map fullConfig = (Map) Yaml.load(yamlFile);
+    Yaml yaml = new Yaml();
+    Map fullConfig = (Map) yaml.load(new FileInputStream(yamlFile));
     Map testConfig = (Map)fullConfig.get("tests");
     return (List)testConfig.get(testType);
   }