Integrate further with ooni/probe-engine: episode two

go.mod

@@ -16,8 +16,7 @@ require (
 	github.com/mattn/go-colorable v0.0.9
 	github.com/mattn/go-isatty v0.0.7 // indirect
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect
-	github.com/measurement-kit/go-measurement-kit v0.0.0-20190521082856-635e836bbb9d
-	github.com/ooni/probe-engine v0.0.0-20190522105151-0c60545f09ea
+	github.com/ooni/probe-engine v0.0.0-20190604110740-9bd31ae52f30
 	github.com/pkg/errors v0.8.1
 	github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051
 	github.com/ziutek/mymysql v1.5.4 // indirect

go.sum

@@ -23,13 +23,14 @@ github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY
 github.com/aristanetworks/goarista v0.0.0-20190514202536-8f808a500156/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ=
 github.com/armon/go-proxyproto v0.0.0-20190211145416-68259f75880e/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/avast/retry-go v2.3.0+incompatible h1:GdXHi3qw0JvbR1Wg1Hr/kx0b6lS36xfypCP4VpZARm4=
+github.com/avast/retry-go v2.3.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/creack/goselect v0.1.0/go.mod h1:gHrIcH/9UZDn2qgeTUeW5K9eZsVYCH6/60J/FHysWyE=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -101,7 +102,7 @@ github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH
 github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk=
 github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao=
 github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58=
-github.com/m-lab/ndt7-client-go v0.0.0-20190516113520-f8aec6e1ef2e/go.mod h1:nD9WTkxP4/mV2ph5uUpVPD2ZOJdZ9QKPTKu3tJGkM9o=
+github.com/m-lab/ndt7-client-go v0.0.0-20190529150833-0f066cf3ab8a/go.mod h1:nD9WTkxP4/mV2ph5uUpVPD2ZOJdZ9QKPTKu3tJGkM9o=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
 github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
@@ -113,16 +114,14 @@ github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/measurement-kit/go-measurement-kit v0.0.0-20190521082856-635e836bbb9d h1:w8l3iUS/zDNcNQwios6icSyH3qXrDE+D6k1kSob0RnU=
-github.com/measurement-kit/go-measurement-kit v0.0.0-20190521082856-635e836bbb9d/go.mod h1:n4Mki43BUXopDPfUkgry9vmSzwZ7dVL5Mx5mRUCWKEw=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/ooni/probe-engine v0.0.0-20190522105151-0c60545f09ea h1:Dx1uyp66YkHDR42r66cA1GlCp4y5hF2fuW+apL9/oV0=
-github.com/ooni/probe-engine v0.0.0-20190522105151-0c60545f09ea/go.mod h1:o6RJkZwHVn6GP7xMLVdRsd4pOxXeo1tJKK29TKKiRhA=
+github.com/ooni/probe-engine v0.0.0-20190604110740-9bd31ae52f30 h1:lzjIBgUmynzfwDKtfA64zEsz0qwokhnS07GiqLoSgIw=
+github.com/ooni/probe-engine v0.0.0-20190604110740-9bd31ae52f30/go.mod h1:A6rE7/l6Gm6M4uCoZmHvgS+JdSmwebgJvYYwlcEg5Eo=
 github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8=
 github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE=
 github.com/oschwald/maxminddb-golang v1.3.0 h1:oTh8IBSj10S5JNlUDg5WjJ1QdBMdeaZIkPEVfESSWgE=
@@ -155,7 +154,6 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
@@ -168,7 +166,6 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b h1:Elez2XeF2p9uyVj0yEUDqQ56NFcDtcBNkYP7yv8YbUE=
 golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=

internal/cli/run/run.go

@@ -1,6 +1,7 @@
 package run
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"strings"
@@ -95,6 +96,22 @@ func init() {
 			log.WithError(err).Error("Failed to create the network row")
 			return err
 		}
+		if ctx.Config.Advanced.BouncerURL != "" {
+			ctx.Session.SetAvailableHTTPSBouncer(ctx.Config.Advanced.BouncerURL)
+		}
+		if err := ctx.Session.MaybeLookupBackends(context.Background()); err != nil {
+			log.WithError(err).Warn("Failed to discover available test helpers")
+			// Rationale for falling through: some tests may be able to complete
+			// with no test helpers, so stopping may be excessive here.
+		}
+		if ctx.Config.Sharing.UploadResults {
+			if ctx.Config.Advanced.CollectorURL != "" {
+				ctx.Session.SetAvailableHTTPSCollector(ctx.Config.Advanced.CollectorURL)
+			} else if err := ctx.Session.MaybeLookupCollectors(context.Background()); err != nil {
+				log.WithError(err).Error("Failed to discover available collectors")
+				return err
+			}
+		}
 
 		if *nettestGroup == "" {
 			log.Infof("Running %s tests", color.BlueString("all"))

internal/enginex/enginex.go

@@ -0,0 +1,38 @@
+// Package enginex contains ooni/probe-engine extensions.
+package enginex
+
+import (
+	"encoding/json"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-engine/model"
+)
+
+// Logger is the logger used by the engine.
+var Logger = log.WithFields(log.Fields{
+  "type": "engine",
+})
+
+// MakeGenericTestKeys casts the m.TestKeys to a map[string]interface{}.
+//
+// Ideally, all tests should have a clear Go structure, well defined, that
+// will be stored in m.TestKeys as an interface. This is not already the
+// case and it's just valid for tests written in Go. Until all tests will
+// be written in Go, we'll keep this glue here to make sure we convert from
+// the engine format to the cli format.
+//
+// This function will first attempt to cast directly to map[string]interface{},
+// which is possible for MK tests, and then use JSON serialization and
+// de-serialization only if that's required.
+func MakeGenericTestKeys(m model.Measurement) (map[string]interface{}, error) {
+	if result, ok := m.TestKeys.(map[string]interface{}); ok {
+		return result, nil
+	}
+	data, err := json.Marshal(m.TestKeys)
+	if err != nil {
+		return nil, err
+	}
+	var result map[string]interface{}
+	err = json.Unmarshal(data, &result)
+	return result, err
+}

internal/log/handlers/cli/cli.go

@@ -104,6 +104,9 @@ func logTable(w io.Writer, f log.Fields) error {
 // TypedLog is used for handling special "typed" logs to the CLI
 func (h *Handler) TypedLog(t string, e *log.Entry) error {
 	switch t {
+	case "engine":
+		fmt.Fprintf(h.Writer, "[engine] %s\n", e.Message)
+		return nil
 	case "progress":
 		perc := e.Fields.Get("percentage").(float64) * 100
 		s := fmt.Sprintf("   %s %-25s",

nettests/im/facebook_messenger.go

@@ -1,8 +1,8 @@
 package im
 
 import (
-	"github.com/measurement-kit/go-measurement-kit"
 	"github.com/ooni/probe-cli/nettests"
+	"github.com/ooni/probe-engine/experiment/fbmessenger"
 )
 
 // FacebookMessenger test implementation
@@ -11,9 +11,10 @@ type FacebookMessenger struct {
 
 // Run starts the test
 func (h FacebookMessenger) Run(ctl *nettests.Controller) error {
-	mknt := mk.NewNettest("FacebookMessenger")
-	ctl.Init(mknt)
-	return mknt.Run()
+	experiment := fbmessenger.NewExperiment(ctl.Ctx.Session, fbmessenger.Config{
+		LogLevel: "INFO",
+	})
+	return ctl.Run(experiment, []string{""})
 }
 
 // FacebookMessengerTestKeys for the test

nettests/im/telegram.go

@@ -1,8 +1,8 @@
 package im
 
 import (
-	"github.com/measurement-kit/go-measurement-kit"
 	"github.com/ooni/probe-cli/nettests"
+	"github.com/ooni/probe-engine/experiment/telegram"
 )
 
 // Telegram test implementation
@@ -11,9 +11,10 @@ type Telegram struct {
 
 // Run starts the test
 func (h Telegram) Run(ctl *nettests.Controller) error {
-	mknt := mk.NewNettest("Telegram")
-	ctl.Init(mknt)
-	return mknt.Run()
+	experiment := telegram.NewExperiment(ctl.Ctx.Session, telegram.Config{
+		LogLevel: "INFO",
+	})
+	return ctl.Run(experiment, []string{""})
 }
 
 // TelegramTestKeys for the test

nettests/im/whatsapp.go

@@ -1,8 +1,8 @@
 package im
 
 import (
-	"github.com/measurement-kit/go-measurement-kit"
 	"github.com/ooni/probe-cli/nettests"
+	"github.com/ooni/probe-engine/experiment/whatsapp"
 )
 
 // WhatsApp test implementation
@@ -11,9 +11,10 @@ type WhatsApp struct {
 
 // Run starts the test
 func (h WhatsApp) Run(ctl *nettests.Controller) error {
-	mknt := mk.NewNettest("Whatsapp")
-	ctl.Init(mknt)
-	return mknt.Run()
+	experiment := whatsapp.NewExperiment(ctl.Ctx.Session, whatsapp.Config{
+		LogLevel: "INFO",
+	})
+	return ctl.Run(experiment, []string{""})
 }
 
 // WhatsAppTestKeys for the test

nettests/middlebox/http_header_field_manipulation.go

@@ -3,8 +3,8 @@ package middlebox
 import (
 	"errors"
 
-	"github.com/measurement-kit/go-measurement-kit"
 	"github.com/ooni/probe-cli/nettests"
+	"github.com/ooni/probe-engine/experiment/hhfm"
 )
 
 // HTTPHeaderFieldManipulation test implementation
@@ -13,9 +13,10 @@ type HTTPHeaderFieldManipulation struct {
 
 // Run starts the test
 func (h HTTPHeaderFieldManipulation) Run(ctl *nettests.Controller) error {
-	mknt := mk.NewNettest("HttpHeaderFieldManipulation")
-	ctl.Init(mknt)
-	return mknt.Run()
+	experiment := hhfm.NewExperiment(ctl.Ctx.Session, hhfm.Config{
+		LogLevel: "INFO",
+	})
+	return ctl.Run(experiment, []string{""})
 }
 
 // HTTPHeaderFieldManipulationTestKeys for the test

nettests/middlebox/http_invalid_request_line.go

@@ -3,8 +3,8 @@ package middlebox
 import (
 	"errors"
 
-	"github.com/measurement-kit/go-measurement-kit"
 	"github.com/ooni/probe-cli/nettests"
+	"github.com/ooni/probe-engine/experiment/hirl"
 )
 
 // HTTPInvalidRequestLine test implementation
@@ -13,9 +13,10 @@ type HTTPInvalidRequestLine struct {
 
 // Run starts the test
 func (h HTTPInvalidRequestLine) Run(ctl *nettests.Controller) error {
-	mknt := mk.NewNettest("HttpInvalidRequestLine")
-	ctl.Init(mknt)
-	return mknt.Run()
+	experiment := hirl.NewExperiment(ctl.Ctx.Session, hirl.Config{
+		LogLevel: "INFO",
+	})
+	return ctl.Run(experiment, []string{""})
 }
 
 // HTTPInvalidRequestLineTestKeys for the test