Move chromiumpreload to chromium/preloadlist and implement reading fr…

…om file or arbitrary URL.

chromiumpreload/chromium.go → chromium/preloadlist/preloadlist.go

@@ -1,4 +1,4 @@
-package chromiumpreload
+package preloadlist
 
 import (
 	"bufio"
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 	"unicode"
@@ -25,10 +26,10 @@ const (
 // The full list contains information about more than just HSTS, but only
 // HSTS-related contents are currently exposed in this struct.
 type PreloadList struct {
-	Entries []PreloadEntry `json:"entries"`
+	Entries []Entry `json:"entries"`
 }
 
-// A PreloadEntry contains the data from an entry in the Chromium
+// A Entry contains the data from an entry in the Chromium
 // Preload list.
 //
 // - Name: The domain name.
@@ -37,66 +38,49 @@ type PreloadList struct {
 //
 // - IncludeSubDomains: If Mode == ForceHTTPS, forces HSTS to apply to
 //   all subdomains.
-type PreloadEntry struct {
+type Entry struct {
 	Name              string `json:"name"`
 	Mode              string `json:"mode"`
 	IncludeSubDomains bool   `json:"include_subdomains"`
 }
 
-// IndexedPreloadEntries is case-insensitive index of
+// IndexedEntries is case-insensitive index of
 // the entries from the given PreloadList.
-type IndexedPreloadEntries struct {
-	index map[string]PreloadEntry
+type IndexedEntries struct {
+	index map[string]Entry
 }
 
 // Index creates an index out of the given list.
-func (p PreloadList) Index() (idx IndexedPreloadEntries) {
-	m := make(map[string]PreloadEntry)
+func (p PreloadList) Index() (idx IndexedEntries) {
+	m := make(map[string]Entry)
 	for _, entry := range p.Entries {
 		d := strings.ToLower(string(entry.Name))
 		m[d] = entry
 	}
-	return IndexedPreloadEntries{
+	return IndexedEntries{
 		index: m,
 	}
 }
 
 // Get acts similar to map access: it returns an entry from the index preload
 // list (if it is present), along with a boolean indicating if the entry is
 // present.
-func (idx IndexedPreloadEntries) Get(domain string) (PreloadEntry, bool) {
+func (idx IndexedEntries) Get(domain string) (Entry, bool) {
 	entry, ok := idx.index[strings.ToLower(domain)]
 	return entry, ok
 }
 
 const (
-	latestChromiumListURL = "https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json?format=TEXT"
+	// LatestChromiumURL is the URL of the latest preload list in the Chromium source.
+	LatestChromiumURL = "https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json?format=TEXT"
 )
 
-// GetLatest retrieves the latest PreloadList from the Chromium source at
-// https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json
-//
-// Note that this list may be up to 12 weeks fresher than the list used
-// by the current stable version of Chrome. See
-// https://www.chromium.org/developers/calendar for a calendar of releases.
-func GetLatest() (PreloadList, error) {
+// Parse reads a preload list in JSON format (with certain possible comments)
+// and returns a parsed version.
+func Parse(r io.Reader) (PreloadList, error) {
 	var list PreloadList
 
-	client := http.Client{
-		Timeout: time.Second * 10,
-	}
-
-	resp, err := client.Get(latestChromiumListURL)
-	if err != nil {
-		return list, err
-	}
-
-	if resp.StatusCode != 200 {
-		return list, fmt.Errorf("status code %d", resp.StatusCode)
-	}
-
-	body := base64.NewDecoder(base64.StdEncoding, resp.Body)
-	jsonBytes, err := removeComments(body)
+	jsonBytes, err := removeComments(r)
 	if err != nil {
 		return list, errors.New("could not decode body")
 	}
@@ -131,3 +115,49 @@ func isCommentLine(line string) bool {
 	trimmed := strings.TrimLeftFunc(line, unicode.IsSpace)
 	return !strings.HasPrefix(trimmed, "//")
 }
+
+// NewFromChromiumURL retrieves the latest PreloadList from a URL that returns
+// the list in base 64.
+func NewFromChromiumURL(u string) (PreloadList, error) {
+	var list PreloadList
+
+	client := http.Client{
+		Timeout: time.Second * 10,
+	}
+
+	resp, err := client.Get(u)
+	if err != nil {
+		return list, err
+	}
+
+	if resp.StatusCode != 200 {
+		return list, fmt.Errorf("status code %d", resp.StatusCode)
+	}
+
+	body := base64.NewDecoder(base64.StdEncoding, resp.Body)
+
+	return Parse(body)
+}
+
+// NewFromLatest retrieves the latest PreloadList from the Chromium source at
+// https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json
+//
+// Note that this list may be up to 12 weeks fresher than the list used
+// by the current stable version of Chrome. See
+// https://www.chromium.org/developers/calendar for a calendar of releases.
+func NewFromLatest() (PreloadList, error) {
+	return NewFromChromiumURL(LatestChromiumURL)
+}
+
+// NewFromFile reads a PreloadList from a JSON file.
+//
+// In a Chromium checkout, the file is at
+// src/net/http/transport_security_state_static.json
+func NewFromFile(fileName string) (PreloadList, error) {
+	b, err := os.Open(fileName)
+	if err != nil {
+		return PreloadList{}, err
+	}
+
+	return Parse(b)
+}

chromium/preloadlist/preloadlist_test.go

@@ -0,0 +1,117 @@
+package preloadlist
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+)
+
+func TestIndexing(t *testing.T) {
+	list := PreloadList{
+		Entries: []Entry{
+			{
+				Name:              "garron.NET",
+				Mode:              "ForceHTTPS",
+				IncludeSubDomains: true,
+			},
+			{
+				Name:              "example.com",
+				Mode:              "",
+				IncludeSubDomains: false,
+			},
+		},
+	}
+
+	idx := list.Index()
+
+	if len(idx.index) != 2 {
+		t.Errorf("Map has the wrong number of entries.")
+	}
+
+	_, ok := idx.Get("example")
+	if ok {
+		t.Errorf("Entry should not be present.")
+	}
+
+	entry, ok := idx.Get("GARRON.net")
+	if !ok {
+		t.Errorf("Entry should be present.")
+	}
+	if entry.Mode != "ForceHTTPS" {
+		t.Errorf("Map has invalid entry.")
+	}
+}
+
+func TestNewFromLatest(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping test to avoid preload list download.")
+	}
+
+	list, err := NewFromLatest()
+	if err != nil {
+		t.Errorf("Could not retrieve preload list.")
+	}
+
+	firstEntry := list.Entries[0]
+	if firstEntry.Name != "pinningtest.appspot.com" {
+		t.Errorf("First entry of preload list does not have the expected name.")
+	}
+}
+
+func TestNewFromChromiumURL(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping test to avoid preload list download.")
+	}
+
+	list, err := NewFromChromiumURL("https://chromium.googlesource.com/chromium/src/+/4f587d7d4532287308715d824d19e7465c9f663e/net/http/transport_security_state_static.json?format=TEXT")
+	if err != nil {
+		t.Error(err)
+	}
+	if len(list.Entries) != 3558 {
+		t.Errorf("Wrong number of entries: %d", len(list.Entries))
+	}
+}
+
+var (
+	testJSON = `{
+  "entries": [
+  	// This is a comment.
+    {"name": "garron.net", "include_subdomains": true, "mode": "force-https"},
+    {"name": "example.com", "include_subdomains": false, "mode": "force-https"},
+    {"name": "gmail.com", "mode": "force-https"},
+
+    // Line above intentionally left blank.
+    {"name": "google.com"},
+    {"name": "pinned.badssl.com", "pins": "pinnymcpinnedkey"}
+  ]
+}`
+	testParsed = PreloadList{Entries: []Entry{
+		{"garron.net", "force-https", true},
+		{"example.com", "force-https", false},
+		{"gmail.com", "force-https", false},
+		{"google.com", "", false},
+		{"pinned.badssl.com", "", false}},
+	}
+)
+
+func TestNewFromFile(t *testing.T) {
+	f, err := ioutil.TempFile("", "preloadlist-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(f.Name())
+
+	if _, err := f.Write([]byte(testJSON)); err != nil {
+		t.Fatal(err)
+	}
+
+	list, err := NewFromFile(f.Name())
+	if err != nil {
+		t.Fatalf("Could not read preload list. %s", err)
+	}
+
+	if !reflect.DeepEqual(list, testParsed) {
+		t.Errorf("Parsed list does not match expected. %#v", list)
+	}
+}

cmd/hstspreload/main.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 
 	"github.com/chromium/hstspreload"
-	"github.com/chromium/hstspreload/chromiumpreload"
+	"github.com/chromium/hstspreload/chromium/preloadlist"
 )
 
 func printHelp() {
@@ -82,7 +82,7 @@ func main() {
 		header, issues = removableDomain(args[1])
 
 	case "status":
-		l, err := chromiumpreload.GetLatest()
+		l, err := preloadlist.NewFromLatest()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s\n", err)
 		}

test_list.json

@@ -0,0 +1,9 @@
+{
+  "entries": [
+    {"name": "garron.net", "include_subdomains": true, "mode": "force-https"},
+    {"name": "example.com", "include_subdomains": false, "mode": "force-https"},
+    {"name": "gmail.com", "mode": "force-https"}
+    {"name": "google.com"}
+    {"name": "pinned.badssl.com", "pins": "pinnymcpinnedkey"}
+  ]
+}