Merge branch 'master' into mobile-staging

.github/ISSUE_TEMPLATE/routine-sprint-releases.md

@@ -13,6 +13,7 @@ assignees: bassosimone
 - [ ] Update resources/assets.go
 - [ ] Run go generate ./...
 - [ ] Tag a new version of ooni/probe-engine
+- [ ] Update again version.go to be alpha
 - [ ] Create release at GitHub
 - [ ] Update ooni/probe-engine mobile-staging branch
 - [ ] Pin ooni/probe-cli to ooni/probe-engine

README.md

@@ -73,16 +73,6 @@ pod 'oonimkall', :podspec => 'https://dl.bintray.com/ooni/ios/oonimkall-VERSION.
 Where VERSION is like `2020.03.30-231914` corresponding to the
 time when the build occurred.
 
-## Release procedure
-
-1. make sure that dependencies are up to date
-
-2. make sure that resources are up to date
-
-3. commit, tag, and push
-
-4. create new release on GitHub
-
 ## Updating dependencies
 
 1. update direct dependencies using:

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/Psiphon-Labs/goarista v0.0.0-20160825065156-d002785f4c67 // indirect
 	github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 // indirect
 	github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742 // indirect
-	github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200819184412-10cb0192d244+incompatible
+	github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.13-0.20200924225012-3a44f293ada3+incompatible
 	github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab // indirect
 	github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f // indirect
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
@@ -26,17 +26,19 @@ require (
 	github.com/deckarep/golang-set v0.0.0-20171013212420-1d4478f51bed // indirect
 	github.com/dgraph-io/badger v1.5.4-0.20180815194500-3a87f6d9c273 // indirect
 	github.com/dgryski/go-farm v0.0.0-20180109070241-2de33835d102 // indirect
-	github.com/elazarl/goproxy v0.0.0-20171101143503-a96fa3a31826 // indirect
+	github.com/elazarl/goproxy v0.0.0-20200809112317-0581fc3aee2d // indirect
+	github.com/florianl/go-nfqueue v1.1.1-0.20200829120558-a2f196e98ab0 // indirect
 	github.com/gobwas/glob v0.2.4-0.20180402141543-f00a7392b439 // indirect
-	github.com/google/go-cmp v0.5.1
+	github.com/google/go-cmp v0.5.2
+	github.com/google/gopacket v1.1.19-0.20200831200443-df1bbd09a561 // indirect
 	github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 // indirect
 	github.com/google/martian/v3 v3.0.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
-	github.com/google/uuid v1.1.1
+	github.com/google/uuid v1.1.2
 	github.com/gorilla/websocket v1.4.2
 	github.com/grafov/m3u8 v0.0.0-20171211212457-6ab8f28ed427 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
-	github.com/iancoleman/strcase v0.1.0
+	github.com/iancoleman/strcase v0.1.2
 	github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441 // indirect
 	github.com/marusama/semaphore v0.0.0-20171214154724-565ffd8e868a // indirect
 	github.com/miekg/dns v1.1.31
@@ -48,7 +50,7 @@ require (
 	github.com/redjack/marionette v0.0.0-20180818172807-360dd8f58226 // indirect
 	github.com/refraction-networking/gotapdance v0.0.0-20190909202946-3a6e1938ad70 // indirect
 	github.com/refraction-networking/utls v0.0.0-20200729012536-186025ac7b77 // indirect
-	github.com/rogpeppe/go-internal v1.6.1
+	github.com/rogpeppe/go-internal v1.6.2
 	github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 // indirect
 	github.com/sergeyfrolov/bsbuffer v0.0.0-20180903213811-94e85abb8507 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
@@ -59,6 +61,5 @@ require (
 	go.uber.org/atomic v1.3.3-0.20180806045314-ca680462431f // indirect
 	go.uber.org/multierr v1.1.1-0.20180122172545-ddea229ff1df // indirect
 	go.uber.org/zap v1.9.2-0.20180814183419-67bc79d13d15 // indirect
-	golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc
-	golang.org/x/sys v0.0.0-20200819171115-d785dc25833f // indirect
+	golang.org/x/net v0.0.0-20200927032502-5d4f70055728
 )

internal/httpheader/useragent.go

@@ -4,6 +4,6 @@ package httpheader
 // UserAgent returns the User-Agent header used for measuring.
 func UserAgent() string {
 	// 8.9% as of Aug 21, 2020 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/
-	const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36"
+	const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36"
 	return ua
 }

legacy/netx/resolver_test.go

@@ -3,6 +3,7 @@ package netx_test
 import (
 	"context"
 	"io"
+	"os"
 	"testing"
 	"time"
 
@@ -20,7 +21,7 @@ func testresolverquick(t *testing.T, network, address string) {
 	}
 	addrs, err := resolver.LookupHost(context.Background(), "dns.google.com")
 	if err != nil {
-		t.Fatalf("legacy/netx/resolver_test.go: %+v", err)
+		t.Fatalf("legacy/netx/resolver_test.go: %+v with %s/%s", err, network, address)
 	}
 	if addrs == nil {
 		t.Fatal("expected non-nil addrs here")
@@ -74,18 +75,30 @@ func TestIntegrationNewResolverTCPDomainNoPort(t *testing.T) {
 }
 
 func TestIntegrationNewResolverDoTAddress(t *testing.T) {
+	if os.Getenv("GITHUB_ACTIONS") == "true" {
+		t.Skip("this test is not reliable in GitHub actions")
+	}
 	testresolverquick(t, "dot", "9.9.9.9:853")
 }
 
 func TestIntegrationNewResolverDoTAddressNoPort(t *testing.T) {
+	if os.Getenv("GITHUB_ACTIONS") == "true" {
+		t.Skip("this test is not reliable in GitHub actions")
+	}
 	testresolverquick(t, "dot", "9.9.9.9")
 }
 
 func TestIntegrationNewResolverDoTDomain(t *testing.T) {
+	if os.Getenv("GITHUB_ACTIONS") == "true" {
+		t.Skip("this test is not reliable in GitHub actions")
+	}
 	testresolverquick(t, "dot", "dns.quad9.net:853")
 }
 
 func TestIntegrationNewResolverDoTDomainNoPort(t *testing.T) {
+	if os.Getenv("GITHUB_ACTIONS") == "true" {
+		t.Skip("this test is not reliable in GitHub actions")
+	}
 	testresolverquick(t, "dot", "dns.quad9.net")
 }
 

oonimkall/session.go

@@ -198,6 +198,13 @@ type GeolocateResults struct {
 	Org string
 }
 
+// MaybeUpdateResources ensures that resources are up to date.
+func (sess *Session) MaybeUpdateResources(ctx *Context) error {
+	sess.mtx.Lock()
+	defer sess.mtx.Unlock()
+	return sess.sessp.MaybeUpdateResources(ctx.ctx)
+}
+
 // Geolocate performs a geolocate operation and returns the results. This method
 // is (in Java terminology) synchronized with the session instance.
 func (sess *Session) Geolocate(ctx *Context) (*GeolocateResults, error) {

oonimkall/session_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strings"
 	"testing"
@@ -14,9 +15,9 @@ import (
 	"github.com/ooni/probe-engine/oonimkall"
 )
 
-func NewSession() (*oonimkall.Session, error) {
+func NewSessionWithAssetsDir(assetsDir string) (*oonimkall.Session, error) {
 	return oonimkall.NewSession(&oonimkall.SessionConfig{
-		AssetsDir:        "../testdata/oonimkall/assets",
+		AssetsDir:        assetsDir,
 		ProbeServicesURL: "https://ams-pg.ooni.org/",
 		SoftwareName:     "oonimkall-test",
 		SoftwareVersion:  "0.1.0",
@@ -25,6 +26,10 @@ func NewSession() (*oonimkall.Session, error) {
 	})
 }
 
+func NewSession() (*oonimkall.Session, error) {
+	return NewSessionWithAssetsDir("../testdata/oonimkall/assets")
+}
+
 func TestNewSessionWithInvalidStateDir(t *testing.T) {
 	sess, err := oonimkall.NewSession(&oonimkall.SessionConfig{
 		StateDir: "",
@@ -49,6 +54,24 @@ func TestNewSessionWithMissingSoftwareName(t *testing.T) {
 	}
 }
 
+func TestMaybeUpdateResourcesWithCancelledContext(t *testing.T) {
+	dir, err := ioutil.TempDir("", "xx")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+	sess, err := NewSessionWithAssetsDir(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := sess.NewContext()
+	ctx.Cancel() // cause immediate failure
+	err = sess.MaybeUpdateResources(ctx)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("not the error we expected: %+v", err)
+	}
+}
+
 func ReduceErrorForGeolocate(err error) error {
 	if err == nil {
 		return errors.New("we expected an error here")
@@ -215,7 +238,9 @@ func TestMain(m *testing.M) {
 	// will run and the number of active sessions and cancels will become
 	// balanced. Especially for the number of active cancels, this is an
 	// indication that we've correctly cleaned them up in the session.
-	exitcode := m.Run()
+	if exitcode := m.Run(); exitcode != 0 {
+		os.Exit(exitcode)
+	}
 	for {
 		m, n := oonimkall.ActiveContexts.Load(), oonimkall.ActiveSessions.Load()
 		fmt.Printf("./oonimkall: ActiveContexts: %d; ActiveSessions: %d\n", m, n)
@@ -224,5 +249,5 @@ func TestMain(m *testing.M) {
 		}
 		time.Sleep(1)
 	}
-	os.Exit(exitcode)
+	os.Exit(0)
 }

resources/assets.go

@@ -2,7 +2,7 @@ package resources
 
 const (
 	// Version contains the assets version.
-	Version = 20200821081345
+	Version = 20200929203018
 
 	// ASNDatabaseName is the ASN-DB file name
 	ASNDatabaseName = "asn.mmdb"
@@ -33,18 +33,18 @@ type ResourceInfo struct {
 // All contains info on all known assets.
 var All = map[string]ResourceInfo{
 	"asn.mmdb": {
-		URLPath:  "/ooni/probe-assets/releases/download/20200821081345/asn.mmdb.gz",
-		GzSHA256: "b118504866e29bc3643c2aec54bf7605fdbac153d60491b0df6960b11837be90",
-		SHA256:   "abcaa0a4bc7d9014affd72f501e64a9b708e701fb0733f72706c4c5ce55897d4",
+		URLPath:  "/ooni/probe-assets/releases/download/20200929203018/asn.mmdb.gz",
+		GzSHA256: "abfed7750af355c2e75feed73cb5a4cf44f4ecb9866199900c65a1e3cb58bda9",
+		SHA256:   "3958d1248b13b5aedc5f03e528a3ef2f2ef4ebdc4449bcd6667d03f722988dc9",
 	},
 	"ca-bundle.pem": {
-		URLPath:  "/ooni/probe-assets/releases/download/20200821081345/ca-bundle.pem.gz",
+		URLPath:  "/ooni/probe-assets/releases/download/20200929203018/ca-bundle.pem.gz",
 		GzSHA256: "3a99970bc782e5f7899de9011618bbadde5057d380e8fc7dc58ee96335bc1c30",
 		SHA256:   "2782f0f8e89c786f40240fc1916677be660fb8d8e25dede50c9f6f7b0c2c2178",
 	},
 	"country.mmdb": {
-		URLPath:  "/ooni/probe-assets/releases/download/20200821081345/country.mmdb.gz",
-		GzSHA256: "c594dea05462f8013d8416cdcf65af61e9adb004d1d6aa9971e021f28cba35b7",
-		SHA256:   "4570ac1701e52aa3d30b2512a3297e028cac6b66a5dac10715db4db8dcd84043",
+		URLPath:  "/ooni/probe-assets/releases/download/20200929203018/country.mmdb.gz",
+		GzSHA256: "d76b387d25c5fd408392a28c02938de149116ad7338d4677aec0835d32cd017c",
+		SHA256:   "36bbb5057022934ed51c6fe1e093b42be15388e0adc1760250f030df21f6f501",
 	},
 }

session.go

@@ -416,7 +416,8 @@ func (s *Session) UserAgent() (useragent string) {
 	return
 }
 
-func (s *Session) fetchResourcesIdempotent(ctx context.Context) error {
+// MaybeUpdateResources updates the resources if needed.
+func (s *Session) MaybeUpdateResources(ctx context.Context) error {
 	return (&resources.Client{
 		HTTPClient: s.DefaultHTTPClient(),
 		Logger:     s.logger,
@@ -517,7 +518,7 @@ func (s *Session) LookupLocationContext(ctx context.Context) (out *model.Locatio
 		resolverIP  string = model.DefaultResolverIP
 		resolverOrg string
 	)
-	err = s.fetchResourcesIdempotent(ctx)
+	err = s.MaybeUpdateResources(ctx)
 	runtimex.PanicOnError(err, "s.fetchResourcesIdempotent failed")
 	probeIP, err = s.lookupProbeIP(ctx)
 	runtimex.PanicOnError(err, "s.lookupProbeIP failed")

session_internal_test.go

@@ -10,10 +10,6 @@ func (s *Session) SetAssetsDir(assetsDir string) {
 	s.assetsDir = assetsDir
 }
 
-func (s *Session) FetchResourcesIdempotent(ctx context.Context) error {
-	return s.fetchResourcesIdempotent(ctx)
-}
-
 func (s *Session) GetAvailableProbeServices() []model.Service {
 	return s.getAvailableProbeServices()
 }

session_test.go

@@ -317,7 +317,7 @@ func TestIntegrationSessionDownloadResources(t *testing.T) {
 	sess := newSessionForTestingNoLookups(t)
 	defer sess.Close()
 	sess.SetAssetsDir(tmpdir)
-	err = sess.FetchResourcesIdempotent(ctx)
+	err = sess.MaybeUpdateResources(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}

version.go

@@ -1,4 +1,4 @@
 package engine
 
 // Version is the version of the engine
-const Version = "0.17.0-beta"
+const Version = "0.17.0"