Integrate further with ooni/probe-engine: episode two

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/mattn/go-isatty v0.0.7 // indirect
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect
 	github.com/measurement-kit/go-measurement-kit v0.0.0-20190521082856-635e836bbb9d
-	github.com/ooni/probe-engine v0.0.0-20190522105151-0c60545f09ea
+	github.com/ooni/probe-engine v0.0.0-20190524123849-cbb0a491705e
 	github.com/pkg/errors v0.8.1
 	github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051
 	github.com/ziutek/mymysql v1.5.4 // indirect

go.sum

@@ -23,13 +23,14 @@ github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY
 github.com/aristanetworks/goarista v0.0.0-20190514202536-8f808a500156/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
 github.com/armon/go-proxyproto v0.0.0-20190211145416-68259f75880e/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/avast/retry-go v2.3.0+incompatible h1:GdXHi3qw0JvbR1Wg1Hr/kx0b6lS36xfypCP4VpZARm4=
+github.com/avast/retry-go v2.3.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/creack/goselect v0.1.0/go.mod h1:gHrIcH/9UZDn2qgeTUeW5K9eZsVYCH6/60J/FHysWyE=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -121,8 +122,8 @@ github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/ooni/probe-engine v0.0.0-20190522105151-0c60545f09ea h1:Dx1uyp66YkHDR42r66cA1GlCp4y5hF2fuW+apL9/oV0=
-github.com/ooni/probe-engine v0.0.0-20190522105151-0c60545f09ea/go.mod h1:o6RJkZwHVn6GP7xMLVdRsd4pOxXeo1tJKK29TKKiRhA=
+github.com/ooni/probe-engine v0.0.0-20190524123849-cbb0a491705e h1:sbd2PAJbbmJhg7npgbOtStkKqtdsQ2WroNI7KoKnPMw=
+github.com/ooni/probe-engine v0.0.0-20190524123849-cbb0a491705e/go.mod h1:8DEbYcGXL+ALZcBxAQwXPzsK2UQ/IThJTi5/PReze5Y=
 github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8=
 github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE=
 github.com/oschwald/maxminddb-golang v1.3.0 h1:oTh8IBSj10S5JNlUDg5WjJ1QdBMdeaZIkPEVfESSWgE=
@@ -155,7 +156,6 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
@@ -168,7 +168,6 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b h1:Elez2XeF2p9uyVj0yEUDqQ56NFcDtcBNkYP7yv8YbUE=
 golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=

internal/cli/run/run.go

@@ -1,6 +1,7 @@
 package run
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"strings"
@@ -95,6 +96,17 @@ func init() {
 			log.WithError(err).Error("Failed to create the network row")
 			return err
 		}
+		if ctx.Config.Advanced.BouncerURL != "" {
+			ctx.Session.SetAvailableHTTPSBouncer(ctx.Config.Advanced.BouncerURL)
+		}
+		if ctx.Config.Sharing.UploadResults {
+			if ctx.Config.Advanced.CollectorURL != "" {
+				ctx.Session.SetAvailableHTTPSCollector(ctx.Config.Advanced.CollectorURL)
+			} else if err := ctx.Session.MaybeLookupCollectors(context.Background()); err != nil {
+				log.WithError(err).Error("Failed to discover available collectors")
+				return err
+			}
+		}
 
 		if *nettestGroup == "" {
 			log.Infof("Running %s tests", color.BlueString("all"))

internal/enginex/enginex.go

@@ -0,0 +1,45 @@
+// Package enginex contains ooni/probe-engine extensions.
+package enginex
+
+import (
+	"github.com/fatih/color"
+	"github.com/ooni/probe-engine/log"
+)
+
+// LoggerAdapter exposes an interface compatible with the logger expected
+// by ooni/probe-engine, and forwards real log messages to a secondary logger
+// having the same interface (compatible with apex/log).
+type LoggerAdapter struct {
+	// Logger is the underlying logger
+	log.Logger
+}
+
+// Debug emits a debug message.
+func (la LoggerAdapter) Debug(msg string) {
+	la.Logger.Debug(color.WhiteString("engine") + ": " + msg)
+}
+
+// Debugf formats and emits a debug message.
+func (la LoggerAdapter) Debugf(format string, v ...interface{}) {
+	la.Logger.Debugf(color.WhiteString("engine")+": "+format, v...)
+}
+
+// Info emits an informational message.
+func (la LoggerAdapter) Info(msg string) {
+	la.Logger.Info(color.BlueString("engine") + ": " + msg)
+}
+
+// Infof format and emits an informational message.
+func (la LoggerAdapter) Infof(format string, v ...interface{}) {
+	la.Logger.Infof(color.BlueString("engine")+": "+format, v...)
+}
+
+// Warn emits a warning message.
+func (la LoggerAdapter) Warn(msg string) {
+	la.Logger.Warn(color.RedString("engine") + ": " + msg)
+}
+
+// Warnf formats and emits a warning message.
+func (la LoggerAdapter) Warnf(format string, v ...interface{}) {
+	la.Logger.Warnf(color.RedString("engine")+": "+format, v...)
+}

nettests/im/facebook_messenger.go

@@ -1,8 +1,10 @@
 package im
 
 import (
-	"github.com/measurement-kit/go-measurement-kit"
+	"errors"
+
 	"github.com/ooni/probe-cli/nettests"
+	"github.com/ooni/probe-engine/experiment/fbmessenger"
 )
 
 // FacebookMessenger test implementation
@@ -11,9 +13,10 @@ type FacebookMessenger struct {
 
 // Run starts the test
 func (h FacebookMessenger) Run(ctl *nettests.Controller) error {
-	mknt := mk.NewNettest("FacebookMessenger")
-	ctl.Init(mknt)
-	return mknt.Run()
+	experiment := fbmessenger.NewExperiment(
+		ctl.Ctx.Session, fbmessenger.Config{},
+	)
+	return ctl.Run(experiment, []string{""})
 }
 
 // FacebookMessengerTestKeys for the test
@@ -24,7 +27,12 @@ type FacebookMessengerTestKeys struct {
 }
 
 // GetTestKeys generates a summary for a test run
-func (h FacebookMessenger) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+func (h FacebookMessenger) GetTestKeys(otk interface{}) (interface{}, error) {
+	tk, ok := otk.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("Unexpected test keys format")
+	}
+
 	var (
 		dnsBlocking bool
 		tcpBlocking bool

nettests/im/telegram.go

@@ -1,8 +1,10 @@
 package im
 
 import (
-	"github.com/measurement-kit/go-measurement-kit"
+	"errors"
+
 	"github.com/ooni/probe-cli/nettests"
+	"github.com/ooni/probe-engine/experiment/telegram"
 )
 
 // Telegram test implementation
@@ -11,9 +13,10 @@ type Telegram struct {
 
 // Run starts the test
 func (h Telegram) Run(ctl *nettests.Controller) error {
-	mknt := mk.NewNettest("Telegram")
-	ctl.Init(mknt)
-	return mknt.Run()
+	experiment := telegram.NewExperiment(
+		ctl.Ctx.Session, telegram.Config{},
+	)
+	return ctl.Run(experiment, []string{""})
 }
 
 // TelegramTestKeys for the test
@@ -25,7 +28,12 @@ type TelegramTestKeys struct {
 }
 
 // GetTestKeys generates a summary for a test run
-func (h Telegram) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+func (h Telegram) GetTestKeys(otk interface{}) (interface{}, error) {
+	tk, ok := otk.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("Unexpected test keys format")
+	}
+
 	var (
 		tcpBlocking  bool
 		httpBlocking bool

nettests/im/whatsapp.go

@@ -1,8 +1,10 @@
 package im
 
 import (
-	"github.com/measurement-kit/go-measurement-kit"
+	"errors"
+
 	"github.com/ooni/probe-cli/nettests"
+	"github.com/ooni/probe-engine/experiment/whatsapp"
 )
 
 // WhatsApp test implementation
@@ -11,9 +13,10 @@ type WhatsApp struct {
 
 // Run starts the test
 func (h WhatsApp) Run(ctl *nettests.Controller) error {
-	mknt := mk.NewNettest("Whatsapp")
-	ctl.Init(mknt)
-	return mknt.Run()
+	experiment := whatsapp.NewExperiment(
+		ctl.Ctx.Session, whatsapp.Config{},
+	)
+	return ctl.Run(experiment, []string{""})
 }
 
 // WhatsAppTestKeys for the test
@@ -25,7 +28,12 @@ type WhatsAppTestKeys struct {
 }
 
 // GetTestKeys generates a summary for a test run
-func (h WhatsApp) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+func (h WhatsApp) GetTestKeys(otk interface{}) (interface{}, error) {
+	tk, ok := otk.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("Unexpected test keys format")
+	}
+
 	var (
 		webBlocking          bool
 		registrationBlocking bool

nettests/middlebox/http_header_field_manipulation.go

@@ -24,7 +24,12 @@ type HTTPHeaderFieldManipulationTestKeys struct {
 }
 
 // GetTestKeys returns a projection of the tests keys needed for the views
-func (h HTTPHeaderFieldManipulation) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+func (h HTTPHeaderFieldManipulation) GetTestKeys(otk interface{}) (interface{}, error) {
+	tk, ok := otk.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("Unexpected test keys format")
+	}
+
 	testKeys := HTTPHeaderFieldManipulationTestKeys{IsAnomaly: false}
 	tampering, ok := tk["tampering"].(map[string]interface{})
 	if !ok {

nettests/middlebox/http_invalid_request_line.go

@@ -24,7 +24,12 @@ type HTTPInvalidRequestLineTestKeys struct {
 }
 
 // GetTestKeys generates a summary for a test run
-func (h HTTPInvalidRequestLine) GetTestKeys(tk map[string]interface{}) (interface{}, error) {
+func (h HTTPInvalidRequestLine) GetTestKeys(otk interface{}) (interface{}, error) {
+	tk, ok := otk.(map[string]interface{})
+	if !ok {
+		return nil, errors.New("Unexpected test keys format")
+	}
+
 	testKeys := HTTPInvalidRequestLineTestKeys{IsAnomaly: false}
 
 	tampering, ok := tk["tampering"].(bool)