Fix coverage from a bunch of rules

install-dev-dependencies.sh

@@ -4,14 +4,17 @@
 set -o errexit -o xtrace
 if type apt-get >/dev/null ; then
   sudo apt-get install libxml2-dev libxml2-utils libxslt1-dev python-dev \
-    firefox chromium-browser zip sqlite3 python-pip
+    firefox chromium-browser zip sqlite3 python-pip libcurl4-openssl-dev
 elif type brew >/dev/null ; then
   brew install python libxml2 gnu-sed
   if ! echo $PATH | grep -ql /usr/local/bin ; then
     echo '/usr/local/bin not found in $PATH, please add it.'
   fi
 fi
 pip install --user --no-allow-insecure --no-allow-external -r requirements.txt
+cd https-everywhere-checker
+pip install --user -r requirements.txt
+cd -
 # Get the addon SDK submodule
 git submodule init
 git submodule update

src/chrome/content/rules/Royal-Navy.xml

@@ -2,13 +2,18 @@
 	c69011.r11.cf3.rackcdn.com
 
 -->
-<ruleset name="Royal Navy (partial)" platform="mixedcontent">
+<ruleset name="Royal Navy (partial)" platform="mixedcontent" default_off="ruleset coverage test fails">
 
 	<target host="navynews.co.uk" />
 	<target host="www.navynews.co.uk" />
 	<target host="royalnavy.mod.uk" />
 	<target host="www.royalnavy.mod.uk" />
 
+	<test url="http://www.royalnavy.mod.uk/images/" />
+	<test url="http://www.royalnavy.mod.uk/library/" />
+	<test url="http://www.royalnavy.mod.uk/login/" />
+	<test url="http://www.royalnavy.mod.uk/profile/" />
+	<test url="http://www.royalnavy.mod.uk/ScriptCombiner.axd" />
 
 	<securecookie host="^(?:www\.)?navynews\.co\.uk$" name=".*" />
 

src/chrome/content/rules/Shaklee.xml

@@ -17,14 +17,17 @@
 	* Some pages redirect to http
 
 -->
-<ruleset name="Shaklee (partial)">
+<ruleset name="Shaklee (partial)" default_off="Needs test URLs">
 
 	<target host="myshaklee.com" />
-	<target host="*.myshaklee.com" />
+	<target host="www.myshaklee.com" />
+	<target host="member.myshaklee.com" />
 		<exclusion pattern="^http://member\.myshaklee\.com/\w\w/\w\w/article/" />
 	<target host="shaklee.com" />
-	<target host="*.shaklee.com" />
-		<exclusion pattern="^htpp;//shaklee\.com/(?:/\w\w/\w\w/)?(?:$|\?)" />
+	<target host="content.shaklee.com" />
+	<target host="images.shaklee.com" />
+	<target host="www.shaklee.com" />
+		<exclusion pattern="^http://shaklee\.com/(?:/\w\w/\w\w/)?(?:$|\?)" />
 
 
 	<!--	Tracking cookies:
@@ -37,10 +40,7 @@
 	<rule from="^http://(?:www\.)?myshaklee\.com/"
 		to="https://www.myshaklee.com/" />
 
-	<rule from="^http://member\.myshaklee\.com/"
-		to="https://member.myshaklee.com/" />
+	<rule from="^http:"
+		to="https:" />
 
-	<rule from="^http://(content\.|images\.|www\.)?shaklee\.com/"
-		to="https://$1shaklee.com/" />
-
-</ruleset>
+</ruleset>

src/chrome/content/rules/Student_Benefits.ca.xml

@@ -9,7 +9,7 @@
 <ruleset name="Student Benefits.ca">
 
 	<target host="studentbenefits.ca" />
-	<target host="*.studentbenefits.ca" />
+	<target host="www.studentbenefits.ca" />
 
 
 	<securecookie host="^\.studentbenefits\.ca$" name=".+" />

src/chrome/content/rules/TDC.dk.xml

@@ -50,11 +50,21 @@
 <ruleset name="TDC.dk (partial)">
 
 	<target host="tdc.dk" />
-	<target host="*.tdc.dk" />
+	<target host="erhverv.tdc.dk" />
+	<target host="fly.tdc.dk" />
+	<target host="kundeservice.tdc.dk" />
+	<target host="mail.tdc.dk" />
+	<target host="om.tdc.dk" />
+	<target host="privat.tdc.dk" />
+	<target host="privatalarm.tdc.dk" />
+	<target host="selfcare.tdc.dk" />
+	<target host="shop.tdc.dk" />
+	<target host="wholesale.tdc.dk" />
+	<target host="www.tdc.dk" />
+	<target host="forum.tdc.dk" />
 	<target host="tdc-wholesale.com" />
 	<target host="www.tdc-wholesale.com" />
 
-
 	<!--	Set by shop:
 				-->
 	<securecookie host="^\.tdc\.dk$" name="^salg_dwbc$" />
@@ -65,16 +75,16 @@
 	<securecookie host="^(?:privatalarm|shop)\.tdc\.dk$" name=".+" />
 
 
-	<rule from="^http://((?:erhverv|fly|kundeservice|mail|om|privat|privatalarm|selfcare|shop|wholesale|www)\.)?tdc\.dk/"
-		to="https://$1tdc.dk/" />
-
-	<rule from="^http://forum\.tdc\.tk/"
+	<rule from="^http://forum\.tdc\.dk/"
 		to="https://tdc.i.lithium.com/" />
 
 	<!--	- //+ redirects to http://$
 		- \w+ 404s
 				-->
-	<rule from="^http://(?:www\.)?tdc-wholesale\.com/(?:[?.][?./]*)?"
+	<rule from="^http://(?:www\.)?tdc-wholesale\.com/"
 		to="https://wholesale.tdc.dk/nordic/" />
 
+	<rule from="^http:"
+		to="https:" />
+
 </ruleset>

src/chrome/content/rules/ThePermitStore.com.xml

@@ -1,9 +1,10 @@
 <ruleset name="ThePermitStore.com">
 	<target host="thepermitstore.com" />
 	<target host="*.thepermitstore.com" />
+	<test url="http://www.thepermitstore.com" />
 
 	<securecookie host="^(?:[a-zA-Z0-9\-]*\.)?thepermitstore\.com$" name=".+" />
 
-	<rule from="^http://thepermitstore\.com/" to="https://thepermitstore.com/" />
-	<rule from="^http://([a-zA-Z0-9\-]+)\.thepermitstore\.com/" to="https://$1.thepermitstore.com/" />
+	<rule from="^http:"
+		to="https:" />
 </ruleset>

src/chrome/content/rules/University-of-Southern-California.xml

@@ -21,12 +21,15 @@
 <ruleset name="University of Southern California (partial)">
 
 	<target host="usc.edu" />
-	<target host="*.usc.edu" />
-
+	<target host="dornsifecms.usc.edu" />
+	<target host="dornsife.usc.edu" />
+	<target host="www.dornsife.usc.edu" />
+	<target host="news.usc.edu" />
+	<target host="vsoeapp1.vsoe.usc.edu" />
+	<target host="www.usc.edu" />
 
 	<securecookie host="^.*\.usc\.edu$" name=".*" />
 
-
 	<!--	- !www doesn't work over https
 		- Redirects as so	-->
 	<rule from="^http://(?:www\.)?usc\.edu/"
@@ -38,8 +41,7 @@
 	<rule from="^http://(?:www\.)?dornsife\.usc\.edu/"
 		to="https://dornsife.usc.edu/" />
 
-	<!--	www.news doesn't exist.	-->
-	<rule from="^http://(dornsifecms|news|vsoeapp1\.vsoe)\.usc\.edu/"
-		to="https://$1.usc.edu/" />
+	<rule from="^http:"
+		to="https:" />
 
 </ruleset>

src/chrome/content/rules/Van_Lanschot.com.xml

@@ -10,7 +10,6 @@
 -->
 <ruleset name="Van Lanschot.com">
 
-	<target host="vanlanschot.com" />
 	<target host="www.vanlanschot.com" />
 
 
@@ -24,8 +23,7 @@
 
 	<securecookie host="^(?:www\.)?vanlanschot\.com$" name=".+" />
 
-
-	<rule from="^http://(?:www\.)?vanlanschot\.com/"
-		to="https://www.vanlanschot.com/" />
+	<rule from="^http:"
+		to="https:" />
 
 </ruleset>

src/chrome/content/rules/WHATWG.org.xml

@@ -30,10 +30,21 @@
 <ruleset name="WHATWG.org">
 
 	<target host="whatwg.org" />
-	<target host="*.whatwg.org" />
-
-
-	<rule from="^http://((?:developers|html-differences|images|resources|\w+\.spec|wiki|www)\.)?whatwg\.org/"
-		to="https://$1whatwg.org/" />
+	<target host="developers.whatwg.org" />
+	<target host="html-differences.whatwg.org" />
+	<target host="images.whatwg.org" />
+	<target host="resources.whatwg.org" />
+	<target host="*.spec.whatwg.org" />
+	<target host="wiki.whatwg.org" />
+	<target host="www.whatwg.org" />
+
+	<test url="http://html.spec.whatwg.org/" />
+	<test url="http://fetch.spec.whatwg.org/" />
+
+	<rule from="^http://whatwg\.org/"
+		to="https://whatwg.org/" />
+
+	<rule from="^http:"
+		to="https:" />
 
 </ruleset>

test-ruleset-coverage.sh

@@ -8,10 +8,10 @@ TMP=`mktemp`
 trap 'rm "$TMP"' EXIT
 # Git log gives us all changed files. Pipe that through ls to eliminate files
 # that have been deleted.
-if ! git log --name-only --date=local --since=2015-02-05 --pretty=format: \
+if ! git log --name-only --date=local --since="2015-02-11 20:00" --pretty=format: \
       src/chrome/content/rules/ | sort -u | \
       xargs ls 2>/dev/null | xargs python2.7 https-everywhere-checker/src/https_everywhere_checker/check_rules.py \
-      https-everywhere-checker/checker.config.sample ; then
+      https-everywhere-checker/coverage.checker.config ; then
   echo "Ruleset test coverage was insufficient. Please add <test url=...> tags " \
        "to ruleset with additional HTTP URLs to test rewriting and fetching."
   exit 1

utils/relaxng.xml

@@ -55,16 +55,17 @@
             <!-- The from attribute of a rule must start with a caret.
                  It also must contain two slashes (separating the protocol from
                  the hostname), and at least one other slash (terminating the
-                 hostname, and possible beginning a path). -->
-            <param name="pattern">\^.*//.*/.*</param>
+                 hostname, and possible beginning a path). Alternately it can be
+                 the literal string "^http:". -->
+            <param name="pattern">(\^.*//.*/.*|\^http:)</param>
           </data>
         </attribute>
         <attribute name="to">
           <data type="string">
             <!-- The to attribute of a rule must not contain spaces or
                  backslashes. It also must contain at least one slash after the
-                 hostname. -->
-            <param name="pattern">https?://[^ \\]*/[^ \\]*</param>
+                 hostname. Alternately it can be the literal string "https:"-->
+            <param name="pattern">(https?://[^ \\]*/[^ \\]*|https:)</param>
           </data>
         </attribute>
         <optional>