Use ooni/probe-engine 0.12.0+patches (#133)

* nettests/groups.go: remove redundant struct names * go.mod go.sum: update deps except probe-engine * Update to ooni/probe-engine@e768161 The API has changed. Methods that used to change bits of the session have been removed. Now the session is more immutable than before. As such, we need to completely fill the config before using it. * Set IncludeCountry to always true Co-authored-by: Arturo Filastò <arturo@filasto.net>
ooni · Jun 4, 2020 · b4934b1 · b4934b1
1 parent aebcaca
commit b4934b1
Show file tree

Hide file tree

Showing 19 changed files with 105 additions and 143 deletions.
diff --git a/config/settings.go b/config/settings.go
@@ -42,13 +42,11 @@ type Sharing struct {
 
 // Advanced settings
 type Advanced struct {
-	SendCrashReports bool   `json:"send_crash_reports"`
-	CollectorURL     string `json:"collector_url"`
-	BouncerURL       string `json:"bouncer_url"`
+	SendCrashReports bool `json:"send_crash_reports"`
 }
 
 // Nettests related settings
 type Nettests struct {
-	WebsitesURLLimit int64 `json:"websites_url_limit"`
+	WebsitesURLLimit             int64    `json:"websites_url_limit"`
 	WebsitesEnabledCategoryCodes []string `json:"websites_enabled_category_codes"`
 }
diff --git a/go.mod b/go.mod
@@ -4,14 +4,14 @@ go 1.14
 
 require (
 	github.com/alecthomas/kingpin v2.2.6+incompatible
-	github.com/apex/log v1.1.4
+	github.com/apex/log v1.3.0
 	github.com/fatih/color v1.9.0
 	github.com/getsentry/raven-go v0.0.0-20190419175539-919484f041ea
 	github.com/mattn/go-colorable v0.1.6
-	github.com/ooni/probe-engine v0.11.0
+	github.com/ooni/probe-engine v0.12.1-0.20200529160604-e768161f91db
 	github.com/pkg/errors v0.9.1
 	github.com/rubenv/sql-migrate v0.0.0-20200429072036-ae26b214fa43
 	gopkg.in/AlecAivazis/survey.v1 v1.8.8
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
-	upper.io/db.v3 v3.6.4+incompatible
+	upper.io/db.v3 v3.7.1+incompatible
 )
diff --git a/go.sum b/go.sum
diff --git a/internal/cli/geoip/geoip.go b/internal/cli/geoip/geoip.go
@@ -17,17 +17,23 @@ func init() {
 			return err
 		}
 
-		err = ctx.MaybeLocationLookup()
+		sess, err := ctx.NewSession()
+		if err != nil {
+			return err
+		}
+		defer sess.Close()
+
+		err = sess.MaybeLookupLocation()
 		if err != nil {
 			return err
 		}
 
 		log.WithFields(log.Fields{
 			"type":         "table",
-			"asn":          ctx.Session.ProbeASNString(),
-			"network_name": ctx.Session.ProbeNetworkName(),
-			"country_code": ctx.Session.ProbeCC(),
-			"ip":           ctx.Session.ProbeIP(),
+			"asn":          sess.ProbeASNString(),
+			"network_name": sess.ProbeNetworkName(),
+			"country_code": sess.ProbeCC(),
+			"ip":           sess.ProbeIP(),
 		}).Info("Looked up your location")
 
 		return nil

diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go
@@ -19,6 +19,28 @@ func runNettestGroup(tg string, ctx *ooni.Context, network *database.Network) er
 		return nil
 	}
 
+	sess, err := ctx.NewSession()
+	if err != nil {
+		log.WithError(err).Error("Failed to create a measurement session")
+		return err
+	}
+	defer sess.Close()
+
+	err = sess.MaybeLookupLocation()
+	if err != nil {
+		log.WithError(err).Error("Failed to lookup the location of the probe")
+		return err
+	}
+	network, err = database.CreateNetwork(ctx.DB, sess)
+	if err != nil {
+		log.WithError(err).Error("Failed to create the network row")
+		return err
+	}
+	if err := sess.MaybeLookupBackends(); err != nil {
+		log.WithError(err).Warn("Failed to discover OONI backends")
+		return err
+	}
+
 	group, ok := nettests.NettestGroups[tg]
 	if !ok {
 		log.Errorf("No test group named %s", tg)
@@ -40,7 +62,7 @@ func runNettestGroup(tg string, ctx *ooni.Context, network *database.Network) er
 			break
 		}
 		log.Debugf("Running test %T", nt)
-		ctl := nettests.NewController(nt, ctx, result)
+		ctl := nettests.NewController(nt, ctx, result, sess)
 		ctl.SetNettestIndex(i, len(group.Nettests))
 		if err = nt.Run(ctl); err != nil {
 			log.WithError(err).Errorf("Failed to run %s", group.Label)
@@ -65,8 +87,6 @@ func init() {
 	}
 
 	noCollector := cmd.Flag("no-collector", "Disable uploading measurements to a collector").Bool()
-	collectorURL := cmd.Flag("collector-url", "Specify the address of a custom collector").String()
-	bouncerURL := cmd.Flag("bouncer-url", "Specify the address of a custom bouncer").String()
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
 		var err error
@@ -84,40 +104,6 @@ func init() {
 		if *noCollector == true {
 			ctx.Config.Sharing.UploadResults = false
 		}
-		if *collectorURL != "" {
-			ctx.Config.Advanced.CollectorURL = *collectorURL
-		}
-		if *bouncerURL != "" {
-			ctx.Config.Advanced.BouncerURL = *bouncerURL
-		}
-		log.Debugf("Using collector: %s", ctx.Config.Advanced.CollectorURL)
-		log.Debugf("Using bouncer: %s", ctx.Config.Advanced.CollectorURL)
-
-		err = ctx.MaybeLocationLookup()
-		if err != nil {
-			log.WithError(err).Error("Failed to lookup the location of the probe")
-			return err
-		}
-		network, err = database.CreateNetwork(ctx.DB, ctx.Session)
-		if err != nil {
-			log.WithError(err).Error("Failed to create the network row")
-			return err
-		}
-		if ctx.Config.Advanced.BouncerURL != "" {
-			ctx.Session.AddAvailableHTTPSBouncer(ctx.Config.Advanced.BouncerURL)
-		}
-		if ctx.Config.Sharing.UploadResults && ctx.Config.Advanced.CollectorURL != "" {
-			ctx.Session.AddAvailableHTTPSCollector(ctx.Config.Advanced.CollectorURL)
-		}
-		if err := ctx.Session.MaybeLookupBackends(); err != nil {
-			log.WithError(err).Warn("Failed to discover OONI backends")
-			return err
-		}
-		// Make sure we share what the user wants us to share.
-		ctx.Session.SetIncludeProbeIP(ctx.Config.Sharing.IncludeIP)
-		ctx.Session.SetIncludeProbeASN(ctx.Config.Sharing.IncludeASN)
-		// Always include probe_cc
-		ctx.Session.SetIncludeProbeCC(true)
 		return nil
 	})
 

diff --git a/nettests/dash.go b/nettests/dash.go
@@ -10,7 +10,7 @@ type Dash struct {
 
 // Run starts the test
 func (d Dash) Run(ctl *Controller) error {
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder("dash")
+	builder, err := ctl.Session.NewExperimentBuilder("dash")
 	if err != nil {
 		return err
 	}

diff --git a/nettests/facebook_messenger.go b/nettests/facebook_messenger.go
@@ -6,7 +6,7 @@ type FacebookMessenger struct {
 
 // Run starts the test
 func (h FacebookMessenger) Run(ctl *Controller) error {
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+	builder, err := ctl.Session.NewExperimentBuilder(
 		"facebook_messenger",
 	)
 	if err != nil {

diff --git a/nettests/groups.go b/nettests/groups.go
@@ -8,35 +8,35 @@ type NettestGroup struct {
 
 // NettestGroups that can be run by the user
 var NettestGroups = map[string]NettestGroup{
-	"websites": NettestGroup{
+	"websites": {
 		Label: "Websites",
 		Nettests: []Nettest{
 			WebConnectivity{},
 		},
 	},
-	"performance": NettestGroup{
+	"performance": {
 		Label: "Performance",
 		Nettests: []Nettest{
 			Dash{},
 			NDT{},
 		},
 	},
-	"middlebox": NettestGroup{
+	"middlebox": {
 		Label: "Middleboxes",
 		Nettests: []Nettest{
 			HTTPInvalidRequestLine{},
 			HTTPHeaderFieldManipulation{},
 		},
 	},
-	"im": NettestGroup{
+	"im": {
 		Label: "Instant Messaging",
 		Nettests: []Nettest{
 			FacebookMessenger{},
 			Telegram{},
 			WhatsApp{},
 		},
 	},
-	"circumvention": NettestGroup{
+	"circumvention": {
 		Label: "Circumvention Tools",
 		Nettests: []Nettest{
 			Psiphon{},

diff --git a/nettests/http_header_field_manipulation.go b/nettests/http_header_field_manipulation.go
@@ -10,7 +10,7 @@ type HTTPHeaderFieldManipulation struct {
 
 // Run starts the test
 func (h HTTPHeaderFieldManipulation) Run(ctl *Controller) error {
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+	builder, err := ctl.Session.NewExperimentBuilder(
 		"http_header_field_manipulation",
 	)
 	if err != nil {

diff --git a/nettests/http_invalid_request_line.go b/nettests/http_invalid_request_line.go
@@ -10,7 +10,7 @@ type HTTPInvalidRequestLine struct {
 
 // Run starts the test
 func (h HTTPInvalidRequestLine) Run(ctl *Controller) error {
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+	builder, err := ctl.Session.NewExperimentBuilder(
 		"http_invalid_request_line",
 	)
 	if err != nil {

diff --git a/nettests/ndt.go b/nettests/ndt.go
@@ -11,7 +11,7 @@ type NDT struct {
 // Run starts the test
 func (n NDT) Run(ctl *Controller) error {
 	// Since 2020-03-18 probe-engine exports v7 as "ndt".
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder("ndt")
+	builder, err := ctl.Session.NewExperimentBuilder("ndt")
 	if err != nil {
 		return err
 	}

diff --git a/nettests/nettests.go b/nettests/nettests.go
@@ -23,18 +23,21 @@ type Nettest interface {
 }
 
 // NewController creates a nettest controller
-func NewController(nt Nettest, ctx *ooni.Context, res *database.Result) *Controller {
+func NewController(
+	nt Nettest, ctx *ooni.Context, res *database.Result, sess *engine.Session) *Controller {
 	return &Controller{
-		Ctx: ctx,
-		nt:  nt,
-		res: res,
+		Ctx:     ctx,
+		nt:      nt,
+		res:     res,
+		Session: sess,
 	}
 }
 
 // Controller is passed to the run method of every Nettest
 // each nettest instance has one controller
 type Controller struct {
 	Ctx         *ooni.Context
+	Session     *engine.Session
 	res         *database.Result
 	nt          Nettest
 	ntCount     int

diff --git a/nettests/nettests_test.go b/nettests/nettests_test.go
@@ -34,7 +34,11 @@ func TestCreateContext(t *testing.T) {
 
 func TestRun(t *testing.T) {
 	ctx := newTestingContext(t)
-	network, err := database.CreateNetwork(ctx.DB, ctx.Session)
+	sess, err := ctx.NewSession()
+	if err != nil {
+		t.Fatal(err)
+	}
+	network, err := database.CreateNetwork(ctx.DB, sess)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -43,6 +47,6 @@ func TestRun(t *testing.T) {
 		t.Fatal(err)
 	}
 	nt := HTTPInvalidRequestLine{}
-	ctl := NewController(nt, ctx, res)
+	ctl := NewController(nt, ctx, res, sess)
 	nt.Run(ctl)
 }
diff --git a/nettests/psiphon.go b/nettests/psiphon.go
@@ -8,7 +8,7 @@ type Psiphon struct {
 
 // Run starts the test
 func (h Psiphon) Run(ctl *Controller) error {
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+	builder, err := ctl.Session.NewExperimentBuilder(
 		"psiphon",
 	)
 	if err != nil {

diff --git a/nettests/telegram.go b/nettests/telegram.go
@@ -6,7 +6,7 @@ type Telegram struct {
 
 // Run starts the test
 func (h Telegram) Run(ctl *Controller) error {
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+	builder, err := ctl.Session.NewExperimentBuilder(
 		"telegram",
 	)
 	if err != nil {

diff --git a/nettests/tor.go b/nettests/tor.go
@@ -6,7 +6,7 @@ type Tor struct {
 
 // Run starts the test
 func (h Tor) Run(ctl *Controller) error {
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+	builder, err := ctl.Session.NewExperimentBuilder(
 		"tor",
 	)
 	if err != nil {

diff --git a/nettests/web_connectivity.go b/nettests/web_connectivity.go
@@ -9,7 +9,7 @@ import (
 func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {
 	var urls []string
 	urlIDMap := make(map[int64]int64)
-	testlist, err := ctl.Ctx.Session.QueryTestListsURLs(&engine.TestListsURLsConfig{
+	testlist, err := ctl.Session.QueryTestListsURLs(&engine.TestListsURLsConfig{
 		Limit:      limit,
 		Categories: categories,
 	})
@@ -44,7 +44,7 @@ func (n WebConnectivity) Run(ctl *Controller) error {
 		return err
 	}
 	ctl.SetInputIdxMap(urlIDMap)
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+	builder, err := ctl.Session.NewExperimentBuilder(
 		"web_connectivity",
 	)
 	if err != nil {

diff --git a/nettests/whatsapp.go b/nettests/whatsapp.go
@@ -6,7 +6,7 @@ type WhatsApp struct {
 
 // Run starts the test
 func (h WhatsApp) Run(ctl *Controller) error {
-	builder, err := ctl.Ctx.Session.NewExperimentBuilder(
+	builder, err := ctl.Session.NewExperimentBuilder(
 		"whatsapp",
 	)
 	if err != nil {