From 13fbe9e9dcdb72d5485cea9ab22c09fc7b3bfe8e Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 4 May 2020 11:14:25 +0200 Subject: [PATCH 1/9] Allow other Go packages to import/use the miniooni CLI Part of https://github.com/ooni/probe-engine/issues/547. --- cmd/miniooni/main.go | 340 +----------------------------------- libminiooni/libminiooni.go | 347 +++++++++++++++++++++++++++++++++++++ 2 files changed, 349 insertions(+), 338 deletions(-) create mode 100644 libminiooni/libminiooni.go diff --git a/cmd/miniooni/main.go b/cmd/miniooni/main.go index bf3b7970..6a6fee43 100644 --- a/cmd/miniooni/main.go +++ b/cmd/miniooni/main.go @@ -1,344 +1,8 @@ // Command miniooni is simple binary for testing purposes. package main -import ( - "errors" - "fmt" - "io" - "io/ioutil" - "net/url" - "os" - "path" - "path/filepath" - "runtime" - "strings" - "time" - - "github.com/apex/log" - engine "github.com/ooni/probe-engine" - "github.com/ooni/probe-engine/internal/humanizex" - "github.com/pborman/getopt/v2" -) - -type options struct { - annotations []string - bouncerURL string - collectorURL string - inputs []string - extraOptions []string - noBouncer bool - noGeoIP bool - noJSON bool - noCollector bool - proxy string - reportfile string - verbose bool -} - -const ( - softwareName = "miniooni" - softwareVersion = "0.1.0-dev" -) - -var ( - globalOptions options - startTime = time.Now() -) - -func init() { - getopt.FlagLong( - &globalOptions.annotations, "annotation", 'A', "Add annotaton", "KEY=VALUE", - ) - getopt.FlagLong( - &globalOptions.bouncerURL, "bouncer", 'b', "Set bouncer base URL", "URL", - ) - getopt.FlagLong( - &globalOptions.collectorURL, "collector", 'c', - "Set collector base URL", "URL", - ) - getopt.FlagLong( - &globalOptions.inputs, "input", 'i', - "Add test-dependent input to the test input", "INPUT", - ) - getopt.FlagLong( - &globalOptions.extraOptions, "option", 'O', - "Pass an option to the experiment", "KEY=VALUE", - ) - getopt.FlagLong( - &globalOptions.noBouncer, "no-bouncer", 0, "Don't use the OONI bouncer", - ) - getopt.FlagLong( - &globalOptions.noGeoIP, "no-geoip", 'g', "Disable GeoIP lookup", - ) - getopt.FlagLong( - &globalOptions.noJSON, "no-json", 'N', "Disable writing to disk", - ) - getopt.FlagLong( - &globalOptions.noCollector, "no-collector", 'n', "Don't use a collector", - ) - getopt.FlagLong( - &globalOptions.proxy, "proxy", 'P', "Set the proxy URL", "URL", - ) - getopt.FlagLong( - &globalOptions.reportfile, "reportfile", 'o', - "Set the report file path", "PATH", - ) - getopt.FlagLong( - &globalOptions.verbose, "verbose", 'v', "Increase verbosity", - ) -} - -func split(s string) (string, string, error) { - v := strings.SplitN(s, "=", 2) - if len(v) != 2 { - return "", "", errors.New("invalid key-value pair") - } - return v[0], v[1], nil -} - -func mustMakeMap(input []string) (output map[string]string) { - output = make(map[string]string) - for _, opt := range input { - key, value, err := split(opt) - if err != nil { - log.WithError(err).Fatal("cannot split key-value pair") - } - output[key] = value - } - return -} - -func mustParseURL(URL string) *url.URL { - rv, err := url.Parse(URL) - if err != nil { - log.WithError(err).Fatal("cannot parse URL") - } - return rv -} - -type logHandler struct { - io.Writer -} - -func (h *logHandler) HandleLog(e *log.Entry) (err error) { - s := fmt.Sprintf("[%14.6f] <%s> %s", - time.Since(startTime).Seconds(), - e.Level, e.Message) - if len(e.Fields) > 0 { - s += fmt.Sprintf(": %+v", e.Fields) - } - s += "\n" - _, err = h.Writer.Write([]byte(s)) - return -} - -// See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da -func gethomedir() string { - if runtime.GOOS == "windows" { - home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") - if home == "" { - home = os.Getenv("USERPROFILE") - } - return home - } - if runtime.GOOS == "linux" { - home := os.Getenv("XDG_CONFIG_HOME") - if home != "" { - return home - } - // fallthrough - } - return os.Getenv("HOME") -} +import "github.com/ooni/probe-engine/libminiooni" func main() { - getopt.Parse() - if len(getopt.Args()) != 1 { - log.Fatal("You must specify the name of the experiment to run") - } - extraOptions := mustMakeMap(globalOptions.extraOptions) - annotations := mustMakeMap(globalOptions.annotations) - - logger := &log.Logger{ - Level: log.InfoLevel, - Handler: &logHandler{Writer: os.Stderr}, - } - if globalOptions.verbose { - logger.Level = log.DebugLevel - } - if globalOptions.reportfile == "" { - globalOptions.reportfile = "report.jsonl" - } - log.Log = logger - - homeDir := gethomedir() - if homeDir == "" { - log.Fatal("home directory is empty") - } - miniooniDir := path.Join(homeDir, ".miniooni") - assetsDir := path.Join(miniooniDir, "assets") - if err := os.MkdirAll(assetsDir, 0700); err != nil { - log.WithError(err).Fatal("cannot create assets directory") - } - log.Infof("miniooni state directory: %s", miniooniDir) - tempDir, err := ioutil.TempDir("", "miniooni") - if err != nil { - log.WithError(err).Fatal("cannot get a temporary directory") - } - log.Debugf("miniooni temporary directory: %s", tempDir) - - var proxyURL *url.URL - if globalOptions.proxy != "" { - proxyURL = mustParseURL(globalOptions.proxy) - } - - kvstore2dir := filepath.Join(miniooniDir, "kvstore2") - kvstore, err := engine.NewFileSystemKVStore(kvstore2dir) - if err != nil { - log.WithError(err).Fatal("cannot create kvstore2 directory") - } - - sess, err := engine.NewSession(engine.SessionConfig{ - AssetsDir: assetsDir, - KVStore: kvstore, - Logger: logger, - ProxyURL: proxyURL, - SoftwareName: softwareName, - SoftwareVersion: softwareVersion, - TempDir: tempDir, - }) - if err != nil { - log.WithError(err).Fatal("cannot create measurement session") - } - defer func() { - sess.Close() - log.Infof("whole session: recv %s, sent %s", - humanizex.SI(sess.KibiBytesReceived()*1024, "byte"), - humanizex.SI(sess.KibiBytesSent()*1024, "byte"), - ) - }() - - if globalOptions.bouncerURL != "" { - sess.AddAvailableHTTPSBouncer(globalOptions.bouncerURL) - } - if globalOptions.collectorURL != "" { - // Implementation note: setting the collector before doing the lookup - // is totally fine because it's a maybe lookup, meaning that any bit - // of information already available will not be looked up again. - sess.AddAvailableHTTPSCollector(globalOptions.collectorURL) - } - - if !globalOptions.noBouncer { - log.Info("Looking up OONI backends; please be patient...") - if err := sess.MaybeLookupBackends(); err != nil { - log.WithError(err).Fatal("cannot lookup OONI backends") - } - } - if !globalOptions.noGeoIP { - log.Info("Looking up your location; please be patient...") - if err := sess.MaybeLookupLocation(); err != nil { - log.WithError(err).Warn("cannot lookup your location") - } else { - log.Infof("- IP: %s", sess.ProbeIP()) - log.Infof("- country: %s", sess.ProbeCC()) - log.Infof( - "- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString(), - ) - log.Infof("- resolver's IP: %s", sess.ResolverIP()) - log.Infof( - "- resolver's network: %s (%s)", - sess.ResolverNetworkName(), - sess.ResolverASNString(), - ) - } - } - - name := getopt.Args()[0] - builder, err := sess.NewExperimentBuilder(name) - if err != nil { - log.WithError(err).Fatal("cannot create experiment builder") - } - if builder.NeedsInput() { - if len(globalOptions.inputs) <= 0 { - log.Info("Fetching test lists") - list, err := sess.QueryTestListsURLs(&engine.TestListsURLsConfig{ - Limit: 16, - }) - if err != nil { - log.WithError(err).Fatal("cannot fetch test lists") - } - for _, entry := range list.Result { - globalOptions.inputs = append(globalOptions.inputs, entry.URL) - } - } - } else if len(globalOptions.inputs) != 0 { - log.Fatal("this experiment does not expect any input") - } else { - // Tests that do not expect input internally require an empty input to run - globalOptions.inputs = append(globalOptions.inputs, "") - } - for key, value := range extraOptions { - if value == "true" || value == "false" { - err := builder.SetOptionBool(key, value == "true") - if err != nil { - log.WithError(err).Fatal("cannot set boolean option") - } - } else { - err := builder.SetOptionString(key, value) - if err != nil { - log.WithError(err).Fatal("cannot set string option") - } - } - } - experiment := builder.NewExperiment() - defer func() { - log.Infof("experiment: recv %s, sent %s", - humanizex.SI(experiment.KibiBytesReceived()*1024, "byte"), - humanizex.SI(experiment.KibiBytesSent()*1024, "byte"), - ) - }() - - if !globalOptions.noCollector { - log.Info("Opening report; please be patient...") - if err := experiment.OpenReport(); err != nil { - log.WithError(err).Fatal("cannot open report") - } - defer experiment.CloseReport() - log.Infof("Report ID: %s", experiment.ReportID()) - } - - inputCount := len(globalOptions.inputs) - inputCounter := 0 - for _, input := range globalOptions.inputs { - inputCounter++ - if input != "" { - log.Infof("[%d/%d] running with input: %s", inputCounter, inputCount, input) - } - measurement, err := experiment.Measure(input) - if err != nil { - log.WithError(err).Warn("measurement failed") - // fallthrough and try to submit what we have anyway. Even if it - // has failed badly, we'd rather see it. - } - measurement.AddAnnotations(annotations) - if !globalOptions.noCollector { - log.Infof("submitting measurement to OONI collector; please be patient...") - if err := experiment.SubmitAndUpdateMeasurement(measurement); err != nil { - log.WithError(err).Warn("submitting measurement failed") - // fallthrough and save to disk what we have. Not saving is - // worst because it means we cannot eventually resubmit. - } - } - if !globalOptions.noJSON { - // Note: must be after submission because submission modifies - // the measurement to include the report ID. - log.Infof("saving measurement to disk") - if err := experiment.SaveMeasurement( - measurement, globalOptions.reportfile, - ); err != nil { - log.WithError(err).Warn("saving measurement failed") - // fallthrough because we're at the bottom of the loop - } - } - } + libminiooni.Main() } diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go new file mode 100644 index 00000000..221b2598 --- /dev/null +++ b/libminiooni/libminiooni.go @@ -0,0 +1,347 @@ +// Package libminiooni implements the cmd/miniooni CLI. +// +// This CLI is compatible with both OONI Probe v2.x and MK v0.10.x. +package libminiooni + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/apex/log" + engine "github.com/ooni/probe-engine" + "github.com/ooni/probe-engine/internal/humanizex" + "github.com/pborman/getopt/v2" +) + +type options struct { + annotations []string + bouncerURL string + collectorURL string + inputs []string + extraOptions []string + noBouncer bool + noGeoIP bool + noJSON bool + noCollector bool + proxy string + reportfile string + verbose bool +} + +const ( + softwareName = "miniooni" + softwareVersion = "0.1.0-dev" +) + +var ( + globalOptions options + startTime = time.Now() +) + +func init() { + getopt.FlagLong( + &globalOptions.annotations, "annotation", 'A', "Add annotaton", "KEY=VALUE", + ) + getopt.FlagLong( + &globalOptions.bouncerURL, "bouncer", 'b', "Set bouncer base URL", "URL", + ) + getopt.FlagLong( + &globalOptions.collectorURL, "collector", 'c', + "Set collector base URL", "URL", + ) + getopt.FlagLong( + &globalOptions.inputs, "input", 'i', + "Add test-dependent input to the test input", "INPUT", + ) + getopt.FlagLong( + &globalOptions.extraOptions, "option", 'O', + "Pass an option to the experiment", "KEY=VALUE", + ) + getopt.FlagLong( + &globalOptions.noBouncer, "no-bouncer", 0, "Don't use the OONI bouncer", + ) + getopt.FlagLong( + &globalOptions.noGeoIP, "no-geoip", 'g', "Disable GeoIP lookup", + ) + getopt.FlagLong( + &globalOptions.noJSON, "no-json", 'N', "Disable writing to disk", + ) + getopt.FlagLong( + &globalOptions.noCollector, "no-collector", 'n', "Don't use a collector", + ) + getopt.FlagLong( + &globalOptions.proxy, "proxy", 'P', "Set the proxy URL", "URL", + ) + getopt.FlagLong( + &globalOptions.reportfile, "reportfile", 'o', + "Set the report file path", "PATH", + ) + getopt.FlagLong( + &globalOptions.verbose, "verbose", 'v', "Increase verbosity", + ) +} + +func split(s string) (string, string, error) { + v := strings.SplitN(s, "=", 2) + if len(v) != 2 { + return "", "", errors.New("invalid key-value pair") + } + return v[0], v[1], nil +} + +func mustMakeMap(input []string) (output map[string]string) { + output = make(map[string]string) + for _, opt := range input { + key, value, err := split(opt) + if err != nil { + log.WithError(err).Fatal("cannot split key-value pair") + } + output[key] = value + } + return +} + +func mustParseURL(URL string) *url.URL { + rv, err := url.Parse(URL) + if err != nil { + log.WithError(err).Fatal("cannot parse URL") + } + return rv +} + +type logHandler struct { + io.Writer +} + +func (h *logHandler) HandleLog(e *log.Entry) (err error) { + s := fmt.Sprintf("[%14.6f] <%s> %s", + time.Since(startTime).Seconds(), + e.Level, e.Message) + if len(e.Fields) > 0 { + s += fmt.Sprintf(": %+v", e.Fields) + } + s += "\n" + _, err = h.Writer.Write([]byte(s)) + return +} + +// See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da +func gethomedir() string { + if runtime.GOOS == "windows" { + home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") + if home == "" { + home = os.Getenv("USERPROFILE") + } + return home + } + if runtime.GOOS == "linux" { + home := os.Getenv("XDG_CONFIG_HOME") + if home != "" { + return home + } + // fallthrough + } + return os.Getenv("HOME") +} + +// Main is the main function of miniooni +func Main() { + getopt.Parse() + if len(getopt.Args()) != 1 { + log.Fatal("You must specify the name of the experiment to run") + } + extraOptions := mustMakeMap(globalOptions.extraOptions) + annotations := mustMakeMap(globalOptions.annotations) + + logger := &log.Logger{ + Level: log.InfoLevel, + Handler: &logHandler{Writer: os.Stderr}, + } + if globalOptions.verbose { + logger.Level = log.DebugLevel + } + if globalOptions.reportfile == "" { + globalOptions.reportfile = "report.jsonl" + } + log.Log = logger + + homeDir := gethomedir() + if homeDir == "" { + log.Fatal("home directory is empty") + } + miniooniDir := path.Join(homeDir, ".miniooni") + assetsDir := path.Join(miniooniDir, "assets") + if err := os.MkdirAll(assetsDir, 0700); err != nil { + log.WithError(err).Fatal("cannot create assets directory") + } + log.Infof("miniooni state directory: %s", miniooniDir) + tempDir, err := ioutil.TempDir("", "miniooni") + if err != nil { + log.WithError(err).Fatal("cannot get a temporary directory") + } + log.Debugf("miniooni temporary directory: %s", tempDir) + + var proxyURL *url.URL + if globalOptions.proxy != "" { + proxyURL = mustParseURL(globalOptions.proxy) + } + + kvstore2dir := filepath.Join(miniooniDir, "kvstore2") + kvstore, err := engine.NewFileSystemKVStore(kvstore2dir) + if err != nil { + log.WithError(err).Fatal("cannot create kvstore2 directory") + } + + sess, err := engine.NewSession(engine.SessionConfig{ + AssetsDir: assetsDir, + KVStore: kvstore, + Logger: logger, + ProxyURL: proxyURL, + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + TempDir: tempDir, + }) + if err != nil { + log.WithError(err).Fatal("cannot create measurement session") + } + defer func() { + sess.Close() + log.Infof("whole session: recv %s, sent %s", + humanizex.SI(sess.KibiBytesReceived()*1024, "byte"), + humanizex.SI(sess.KibiBytesSent()*1024, "byte"), + ) + }() + + if globalOptions.bouncerURL != "" { + sess.AddAvailableHTTPSBouncer(globalOptions.bouncerURL) + } + if globalOptions.collectorURL != "" { + // Implementation note: setting the collector before doing the lookup + // is totally fine because it's a maybe lookup, meaning that any bit + // of information already available will not be looked up again. + sess.AddAvailableHTTPSCollector(globalOptions.collectorURL) + } + + if !globalOptions.noBouncer { + log.Info("Looking up OONI backends; please be patient...") + if err := sess.MaybeLookupBackends(); err != nil { + log.WithError(err).Fatal("cannot lookup OONI backends") + } + } + if !globalOptions.noGeoIP { + log.Info("Looking up your location; please be patient...") + if err := sess.MaybeLookupLocation(); err != nil { + log.WithError(err).Warn("cannot lookup your location") + } else { + log.Infof("- IP: %s", sess.ProbeIP()) + log.Infof("- country: %s", sess.ProbeCC()) + log.Infof( + "- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString(), + ) + log.Infof("- resolver's IP: %s", sess.ResolverIP()) + log.Infof( + "- resolver's network: %s (%s)", + sess.ResolverNetworkName(), + sess.ResolverASNString(), + ) + } + } + + name := getopt.Args()[0] + builder, err := sess.NewExperimentBuilder(name) + if err != nil { + log.WithError(err).Fatal("cannot create experiment builder") + } + if builder.NeedsInput() { + if len(globalOptions.inputs) <= 0 { + log.Info("Fetching test lists") + list, err := sess.QueryTestListsURLs(&engine.TestListsURLsConfig{ + Limit: 16, + }) + if err != nil { + log.WithError(err).Fatal("cannot fetch test lists") + } + for _, entry := range list.Result { + globalOptions.inputs = append(globalOptions.inputs, entry.URL) + } + } + } else if len(globalOptions.inputs) != 0 { + log.Fatal("this experiment does not expect any input") + } else { + // Tests that do not expect input internally require an empty input to run + globalOptions.inputs = append(globalOptions.inputs, "") + } + for key, value := range extraOptions { + if value == "true" || value == "false" { + err := builder.SetOptionBool(key, value == "true") + if err != nil { + log.WithError(err).Fatal("cannot set boolean option") + } + } else { + err := builder.SetOptionString(key, value) + if err != nil { + log.WithError(err).Fatal("cannot set string option") + } + } + } + experiment := builder.NewExperiment() + defer func() { + log.Infof("experiment: recv %s, sent %s", + humanizex.SI(experiment.KibiBytesReceived()*1024, "byte"), + humanizex.SI(experiment.KibiBytesSent()*1024, "byte"), + ) + }() + + if !globalOptions.noCollector { + log.Info("Opening report; please be patient...") + if err := experiment.OpenReport(); err != nil { + log.WithError(err).Fatal("cannot open report") + } + defer experiment.CloseReport() + log.Infof("Report ID: %s", experiment.ReportID()) + } + + inputCount := len(globalOptions.inputs) + inputCounter := 0 + for _, input := range globalOptions.inputs { + inputCounter++ + if input != "" { + log.Infof("[%d/%d] running with input: %s", inputCounter, inputCount, input) + } + measurement, err := experiment.Measure(input) + if err != nil { + log.WithError(err).Warn("measurement failed") + // fallthrough and try to submit what we have anyway. Even if it + // has failed badly, we'd rather see it. + } + measurement.AddAnnotations(annotations) + if !globalOptions.noCollector { + log.Infof("submitting measurement to OONI collector; please be patient...") + if err := experiment.SubmitAndUpdateMeasurement(measurement); err != nil { + log.WithError(err).Warn("submitting measurement failed") + // fallthrough and save to disk what we have. Not saving is + // worst because it means we cannot eventually resubmit. + } + } + if !globalOptions.noJSON { + // Note: must be after submission because submission modifies + // the measurement to include the report ID. + log.Infof("saving measurement to disk") + if err := experiment.SaveMeasurement( + measurement, globalOptions.reportfile, + ); err != nil { + log.WithError(err).Warn("saving measurement failed") + // fallthrough because we're at the bottom of the loop + } + } + } +} From d7d928f48122868fc77b479bbdc4db6c2cd467c5 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 4 May 2020 11:31:14 +0200 Subject: [PATCH 2/9] libminiooni: add syntactic sugar to handle errors This reduces the number of unnecessary code paths to be tested. --- libminiooni/libminiooni.go | 112 ++++++++++++++----------------------- 1 file changed, 41 insertions(+), 71 deletions(-) diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go index 221b2598..f640038d 100644 --- a/libminiooni/libminiooni.go +++ b/libminiooni/libminiooni.go @@ -98,13 +98,23 @@ func split(s string) (string, string, error) { return v[0], v[1], nil } +func fatalOnError(err error, msg string) { + if err != nil { + log.WithError(err).Fatal(msg) + } +} + +func warnOnError(err error, msg string) { + if err != nil { + log.WithError(err).Warn(msg) + } +} + func mustMakeMap(input []string) (output map[string]string) { output = make(map[string]string) for _, opt := range input { key, value, err := split(opt) - if err != nil { - log.WithError(err).Fatal("cannot split key-value pair") - } + fatalOnError(err, "cannot split key-value pair") output[key] = value } return @@ -112,9 +122,7 @@ func mustMakeMap(input []string) (output map[string]string) { func mustParseURL(URL string) *url.URL { rv, err := url.Parse(URL) - if err != nil { - log.WithError(err).Fatal("cannot parse URL") - } + fatalOnError(err, "cannot parse URL") return rv } @@ -162,10 +170,7 @@ func Main() { extraOptions := mustMakeMap(globalOptions.extraOptions) annotations := mustMakeMap(globalOptions.annotations) - logger := &log.Logger{ - Level: log.InfoLevel, - Handler: &logHandler{Writer: os.Stderr}, - } + logger := &log.Logger{Level: log.InfoLevel, Handler: &logHandler{Writer: os.Stderr}} if globalOptions.verbose { logger.Level = log.DebugLevel } @@ -180,14 +185,11 @@ func Main() { } miniooniDir := path.Join(homeDir, ".miniooni") assetsDir := path.Join(miniooniDir, "assets") - if err := os.MkdirAll(assetsDir, 0700); err != nil { - log.WithError(err).Fatal("cannot create assets directory") - } + err := os.MkdirAll(assetsDir, 0700) + fatalOnError(err, "cannot create assets directory") log.Infof("miniooni state directory: %s", miniooniDir) tempDir, err := ioutil.TempDir("", "miniooni") - if err != nil { - log.WithError(err).Fatal("cannot get a temporary directory") - } + fatalOnError(err, "cannot get a temporary directory") log.Debugf("miniooni temporary directory: %s", tempDir) var proxyURL *url.URL @@ -197,9 +199,7 @@ func Main() { kvstore2dir := filepath.Join(miniooniDir, "kvstore2") kvstore, err := engine.NewFileSystemKVStore(kvstore2dir) - if err != nil { - log.WithError(err).Fatal("cannot create kvstore2 directory") - } + fatalOnError(err, "cannot create kvstore2 directory") sess, err := engine.NewSession(engine.SessionConfig{ AssetsDir: assetsDir, @@ -210,9 +210,7 @@ func Main() { SoftwareVersion: softwareVersion, TempDir: tempDir, }) - if err != nil { - log.WithError(err).Fatal("cannot create measurement session") - } + fatalOnError(err, "cannot create measurement session") defer func() { sess.Close() log.Infof("whole session: recv %s, sent %s", @@ -233,43 +231,31 @@ func Main() { if !globalOptions.noBouncer { log.Info("Looking up OONI backends; please be patient...") - if err := sess.MaybeLookupBackends(); err != nil { - log.WithError(err).Fatal("cannot lookup OONI backends") - } + err := sess.MaybeLookupBackends() + fatalOnError(err, "cannot lookup OONI backends") } if !globalOptions.noGeoIP { log.Info("Looking up your location; please be patient...") - if err := sess.MaybeLookupLocation(); err != nil { - log.WithError(err).Warn("cannot lookup your location") - } else { - log.Infof("- IP: %s", sess.ProbeIP()) - log.Infof("- country: %s", sess.ProbeCC()) - log.Infof( - "- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString(), - ) - log.Infof("- resolver's IP: %s", sess.ResolverIP()) - log.Infof( - "- resolver's network: %s (%s)", - sess.ResolverNetworkName(), - sess.ResolverASNString(), - ) - } + err := sess.MaybeLookupLocation() + warnOnError(err, "cannot lookup your location") + log.Infof("- IP: %s", sess.ProbeIP()) + log.Infof("- country: %s", sess.ProbeCC()) + log.Infof("- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString()) + log.Infof("- resolver's IP: %s", sess.ResolverIP()) + log.Infof("- resolver's network: %s (%s)", sess.ResolverNetworkName(), + sess.ResolverASNString()) } name := getopt.Args()[0] builder, err := sess.NewExperimentBuilder(name) - if err != nil { - log.WithError(err).Fatal("cannot create experiment builder") - } + fatalOnError(err, "cannot create experiment builder") if builder.NeedsInput() { if len(globalOptions.inputs) <= 0 { log.Info("Fetching test lists") list, err := sess.QueryTestListsURLs(&engine.TestListsURLsConfig{ Limit: 16, }) - if err != nil { - log.WithError(err).Fatal("cannot fetch test lists") - } + fatalOnError(err, "cannot fetch test lists") for _, entry := range list.Result { globalOptions.inputs = append(globalOptions.inputs, entry.URL) } @@ -283,14 +269,10 @@ func Main() { for key, value := range extraOptions { if value == "true" || value == "false" { err := builder.SetOptionBool(key, value == "true") - if err != nil { - log.WithError(err).Fatal("cannot set boolean option") - } + fatalOnError(err, "cannot set boolean option") } else { err := builder.SetOptionString(key, value) - if err != nil { - log.WithError(err).Fatal("cannot set string option") - } + fatalOnError(err, "cannot set string option") } } experiment := builder.NewExperiment() @@ -303,9 +285,8 @@ func Main() { if !globalOptions.noCollector { log.Info("Opening report; please be patient...") - if err := experiment.OpenReport(); err != nil { - log.WithError(err).Fatal("cannot open report") - } + err := experiment.OpenReport() + fatalOnError(err, "cannot open report") defer experiment.CloseReport() log.Infof("Report ID: %s", experiment.ReportID()) } @@ -318,30 +299,19 @@ func Main() { log.Infof("[%d/%d] running with input: %s", inputCounter, inputCount, input) } measurement, err := experiment.Measure(input) - if err != nil { - log.WithError(err).Warn("measurement failed") - // fallthrough and try to submit what we have anyway. Even if it - // has failed badly, we'd rather see it. - } + warnOnError(err, "measurement failed") measurement.AddAnnotations(annotations) if !globalOptions.noCollector { log.Infof("submitting measurement to OONI collector; please be patient...") - if err := experiment.SubmitAndUpdateMeasurement(measurement); err != nil { - log.WithError(err).Warn("submitting measurement failed") - // fallthrough and save to disk what we have. Not saving is - // worst because it means we cannot eventually resubmit. - } + err := experiment.SubmitAndUpdateMeasurement(measurement) + warnOnError(err, "submitting measurement failed") } if !globalOptions.noJSON { // Note: must be after submission because submission modifies // the measurement to include the report ID. log.Infof("saving measurement to disk") - if err := experiment.SaveMeasurement( - measurement, globalOptions.reportfile, - ); err != nil { - log.WithError(err).Warn("saving measurement failed") - // fallthrough because we're at the bottom of the loop - } + err := experiment.SaveMeasurement(measurement, globalOptions.reportfile) + warnOnError(err, "saving measurement failed") } } } From 7bd69292e471b55b2232e2daaf1715cc2b1e0078 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 4 May 2020 11:33:15 +0200 Subject: [PATCH 3/9] libminiooni: allow a not-particularly-long line --- libminiooni/libminiooni.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go index f640038d..2f4b69a8 100644 --- a/libminiooni/libminiooni.go +++ b/libminiooni/libminiooni.go @@ -131,9 +131,7 @@ type logHandler struct { } func (h *logHandler) HandleLog(e *log.Entry) (err error) { - s := fmt.Sprintf("[%14.6f] <%s> %s", - time.Since(startTime).Seconds(), - e.Level, e.Message) + s := fmt.Sprintf("[%14.6f] <%s> %s", time.Since(startTime).Seconds(), e.Level, e.Message) if len(e.Fields) > 0 { s += fmt.Sprintf(": %+v", e.Fields) } From f4484582137da0dba69b37989afd4d3b4fbd7e8f Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 4 May 2020 11:37:32 +0200 Subject: [PATCH 4/9] libminiooni: more cases where we avoid explicit if statements --- libminiooni/libminiooni.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go index 2f4b69a8..996c9ded 100644 --- a/libminiooni/libminiooni.go +++ b/libminiooni/libminiooni.go @@ -110,6 +110,12 @@ func warnOnError(err error, msg string) { } } +func fatalIfFalse(cond bool, msg string) { + if !cond { + log.Fatal(msg) + } +} + func mustMakeMap(input []string) (output map[string]string) { output = make(map[string]string) for _, opt := range input { @@ -162,9 +168,7 @@ func gethomedir() string { // Main is the main function of miniooni func Main() { getopt.Parse() - if len(getopt.Args()) != 1 { - log.Fatal("You must specify the name of the experiment to run") - } + fatalIfFalse(len(getopt.Args()) == 1, "Missing experiment name") extraOptions := mustMakeMap(globalOptions.extraOptions) annotations := mustMakeMap(globalOptions.annotations) @@ -178,9 +182,7 @@ func Main() { log.Log = logger homeDir := gethomedir() - if homeDir == "" { - log.Fatal("home directory is empty") - } + fatalIfFalse(homeDir != "", "home directory is empty") miniooniDir := path.Join(homeDir, ".miniooni") assetsDir := path.Join(miniooniDir, "assets") err := os.MkdirAll(assetsDir, 0700) From 4974d6cbb5cb77dba455c0106c78b610a3820b45 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 4 May 2020 11:44:11 +0200 Subject: [PATCH 5/9] libminiooni: disable -g and stop if we cannot get the user IP Closes https://github.com/ooni/probe-engine/issues/297 --- libminiooni/libminiooni.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go index 996c9ded..3bfbd85d 100644 --- a/libminiooni/libminiooni.go +++ b/libminiooni/libminiooni.go @@ -70,7 +70,8 @@ func init() { &globalOptions.noBouncer, "no-bouncer", 0, "Don't use the OONI bouncer", ) getopt.FlagLong( - &globalOptions.noGeoIP, "no-geoip", 'g', "Disable GeoIP lookup", + &globalOptions.noGeoIP, "no-geoip", 'g', + "Disable GeoIP lookup (not implemented!)", ) getopt.FlagLong( &globalOptions.noJSON, "no-json", 'N', "Disable writing to disk", @@ -234,17 +235,20 @@ func Main() { err := sess.MaybeLookupBackends() fatalOnError(err, "cannot lookup OONI backends") } - if !globalOptions.noGeoIP { - log.Info("Looking up your location; please be patient...") - err := sess.MaybeLookupLocation() - warnOnError(err, "cannot lookup your location") - log.Infof("- IP: %s", sess.ProbeIP()) - log.Infof("- country: %s", sess.ProbeCC()) - log.Infof("- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString()) - log.Infof("- resolver's IP: %s", sess.ResolverIP()) - log.Infof("- resolver's network: %s (%s)", sess.ResolverNetworkName(), - sess.ResolverASNString()) - } + // See https://github.com/ooni/probe-engine/issues/297 + fatalIfFalse( + globalOptions.noGeoIP == false, + "Sorry, the -g option is not implemented.", + ) + log.Info("Looking up your location; please be patient...") + err = sess.MaybeLookupLocation() + fatalOnError(err, "cannot lookup your location") + log.Infof("- IP: %s", sess.ProbeIP()) + log.Infof("- country: %s", sess.ProbeCC()) + log.Infof("- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString()) + log.Infof("- resolver's IP: %s", sess.ResolverIP()) + log.Infof("- resolver's network: %s (%s)", sess.ResolverNetworkName(), + sess.ResolverASNString()) name := getopt.Args()[0] builder, err := sess.NewExperimentBuilder(name) From 5443fcbf2175745c5970640fe90e293f18f178e9 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 4 May 2020 12:12:09 +0200 Subject: [PATCH 6/9] libminiooni: decouple the real main from globals --- libminiooni/libminiooni.go | 63 ++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go index 3bfbd85d..5ff26410 100644 --- a/libminiooni/libminiooni.go +++ b/libminiooni/libminiooni.go @@ -91,6 +91,13 @@ func init() { ) } +// Main is the main function of miniooni +func Main() { + getopt.Parse() + fatalIfFalse(len(getopt.Args()) == 1, "Missing experiment name") + MainWithConfiguration(getopt.Arg(0), globalOptions) +} + func split(s string) (string, string, error) { v := strings.SplitN(s, "=", 2) if len(v) != 2 { @@ -166,19 +173,18 @@ func gethomedir() string { return os.Getenv("HOME") } -// Main is the main function of miniooni -func Main() { - getopt.Parse() - fatalIfFalse(len(getopt.Args()) == 1, "Missing experiment name") - extraOptions := mustMakeMap(globalOptions.extraOptions) - annotations := mustMakeMap(globalOptions.annotations) +// MainWithConfiguration is the miniooni main with a specific configuration +// represented by the experiment name and the current options. +func MainWithConfiguration(experimentName string, currentOptions options) { + extraOptions := mustMakeMap(currentOptions.extraOptions) + annotations := mustMakeMap(currentOptions.annotations) logger := &log.Logger{Level: log.InfoLevel, Handler: &logHandler{Writer: os.Stderr}} - if globalOptions.verbose { + if currentOptions.verbose { logger.Level = log.DebugLevel } - if globalOptions.reportfile == "" { - globalOptions.reportfile = "report.jsonl" + if currentOptions.reportfile == "" { + currentOptions.reportfile = "report.jsonl" } log.Log = logger @@ -194,8 +200,8 @@ func Main() { log.Debugf("miniooni temporary directory: %s", tempDir) var proxyURL *url.URL - if globalOptions.proxy != "" { - proxyURL = mustParseURL(globalOptions.proxy) + if currentOptions.proxy != "" { + proxyURL = mustParseURL(currentOptions.proxy) } kvstore2dir := filepath.Join(miniooniDir, "kvstore2") @@ -220,24 +226,24 @@ func Main() { ) }() - if globalOptions.bouncerURL != "" { - sess.AddAvailableHTTPSBouncer(globalOptions.bouncerURL) + if currentOptions.bouncerURL != "" { + sess.AddAvailableHTTPSBouncer(currentOptions.bouncerURL) } - if globalOptions.collectorURL != "" { + if currentOptions.collectorURL != "" { // Implementation note: setting the collector before doing the lookup // is totally fine because it's a maybe lookup, meaning that any bit // of information already available will not be looked up again. - sess.AddAvailableHTTPSCollector(globalOptions.collectorURL) + sess.AddAvailableHTTPSCollector(currentOptions.collectorURL) } - if !globalOptions.noBouncer { + if !currentOptions.noBouncer { log.Info("Looking up OONI backends; please be patient...") err := sess.MaybeLookupBackends() fatalOnError(err, "cannot lookup OONI backends") } // See https://github.com/ooni/probe-engine/issues/297 fatalIfFalse( - globalOptions.noGeoIP == false, + currentOptions.noGeoIP == false, "Sorry, the -g option is not implemented.", ) log.Info("Looking up your location; please be patient...") @@ -250,25 +256,24 @@ func Main() { log.Infof("- resolver's network: %s (%s)", sess.ResolverNetworkName(), sess.ResolverASNString()) - name := getopt.Args()[0] - builder, err := sess.NewExperimentBuilder(name) + builder, err := sess.NewExperimentBuilder(experimentName) fatalOnError(err, "cannot create experiment builder") if builder.NeedsInput() { - if len(globalOptions.inputs) <= 0 { + if len(currentOptions.inputs) <= 0 { log.Info("Fetching test lists") list, err := sess.QueryTestListsURLs(&engine.TestListsURLsConfig{ Limit: 16, }) fatalOnError(err, "cannot fetch test lists") for _, entry := range list.Result { - globalOptions.inputs = append(globalOptions.inputs, entry.URL) + currentOptions.inputs = append(currentOptions.inputs, entry.URL) } } - } else if len(globalOptions.inputs) != 0 { + } else if len(currentOptions.inputs) != 0 { log.Fatal("this experiment does not expect any input") } else { // Tests that do not expect input internally require an empty input to run - globalOptions.inputs = append(globalOptions.inputs, "") + currentOptions.inputs = append(currentOptions.inputs, "") } for key, value := range extraOptions { if value == "true" || value == "false" { @@ -287,7 +292,7 @@ func Main() { ) }() - if !globalOptions.noCollector { + if !currentOptions.noCollector { log.Info("Opening report; please be patient...") err := experiment.OpenReport() fatalOnError(err, "cannot open report") @@ -295,9 +300,9 @@ func Main() { log.Infof("Report ID: %s", experiment.ReportID()) } - inputCount := len(globalOptions.inputs) + inputCount := len(currentOptions.inputs) inputCounter := 0 - for _, input := range globalOptions.inputs { + for _, input := range currentOptions.inputs { inputCounter++ if input != "" { log.Infof("[%d/%d] running with input: %s", inputCounter, inputCount, input) @@ -305,16 +310,16 @@ func Main() { measurement, err := experiment.Measure(input) warnOnError(err, "measurement failed") measurement.AddAnnotations(annotations) - if !globalOptions.noCollector { + if !currentOptions.noCollector { log.Infof("submitting measurement to OONI collector; please be patient...") err := experiment.SubmitAndUpdateMeasurement(measurement) warnOnError(err, "submitting measurement failed") } - if !globalOptions.noJSON { + if !currentOptions.noJSON { // Note: must be after submission because submission modifies // the measurement to include the report ID. log.Infof("saving measurement to disk") - err := experiment.SaveMeasurement(measurement, globalOptions.reportfile) + err := experiment.SaveMeasurement(measurement, currentOptions.reportfile) warnOnError(err, "saving measurement failed") } } From e6dcf40ef7e34b4ed6c4f69a644dd1005a946abe Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 4 May 2020 12:19:19 +0200 Subject: [PATCH 7/9] libminiooni: panic on failure and let cmd/miniooni exit This allows to write some basic tests of libminiooni. --- cmd/miniooni/main.go | 11 ++++++++++- libminiooni/libminiooni.go | 32 +++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/cmd/miniooni/main.go b/cmd/miniooni/main.go index 6a6fee43..be685500 100644 --- a/cmd/miniooni/main.go +++ b/cmd/miniooni/main.go @@ -1,8 +1,17 @@ // Command miniooni is simple binary for testing purposes. package main -import "github.com/ooni/probe-engine/libminiooni" +import ( + "os" + + "github.com/ooni/probe-engine/libminiooni" +) func main() { + defer func() { + if recover() != nil { + os.Exit(1) + } + }() libminiooni.Main() } diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go index 5ff26410..8d844bd0 100644 --- a/libminiooni/libminiooni.go +++ b/libminiooni/libminiooni.go @@ -91,7 +91,23 @@ func init() { ) } -// Main is the main function of miniooni +func fatalWithString(msg string) { + panic(msg) +} + +func fatalIfFalse(cond bool, msg string) { + if !cond { + log.Warn(msg) + panic(msg) + } +} + +// Main is the main function of miniooni. This function parses the command line +// options and uses a global state. Use MainWithConfiguration if you want to avoid +// using any global state and relying on command line options. +// +// This function will panic in case of a fatal error. It is up to you that +// integrate this function to either handle the panic of ignore it. func Main() { getopt.Parse() fatalIfFalse(len(getopt.Args()) == 1, "Missing experiment name") @@ -108,7 +124,8 @@ func split(s string) (string, string, error) { func fatalOnError(err error, msg string) { if err != nil { - log.WithError(err).Fatal(msg) + log.WithError(err).Warn(msg) + panic(msg) } } @@ -118,12 +135,6 @@ func warnOnError(err error, msg string) { } } -func fatalIfFalse(cond bool, msg string) { - if !cond { - log.Fatal(msg) - } -} - func mustMakeMap(input []string) (output map[string]string) { output = make(map[string]string) for _, opt := range input { @@ -175,6 +186,9 @@ func gethomedir() string { // MainWithConfiguration is the miniooni main with a specific configuration // represented by the experiment name and the current options. +// +// This function will panic in case of a fatal error. It is up to you that +// integrate this function to either handle the panic of ignore it. func MainWithConfiguration(experimentName string, currentOptions options) { extraOptions := mustMakeMap(currentOptions.extraOptions) annotations := mustMakeMap(currentOptions.annotations) @@ -270,7 +284,7 @@ func MainWithConfiguration(experimentName string, currentOptions options) { } } } else if len(currentOptions.inputs) != 0 { - log.Fatal("this experiment does not expect any input") + fatalWithString("this experiment does not expect any input") } else { // Tests that do not expect input internally require an empty input to run currentOptions.inputs = append(currentOptions.inputs, "") From ea6820415b3509bfce3d4fac5bb97313f9fa6d46 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 4 May 2020 12:24:22 +0200 Subject: [PATCH 8/9] libminiooni: add basic integration test It just verifies we can run the example experiment without crashing because some options/tables/etc are wrong/uninitalized. This is not much, but better than what we had before. --- libminiooni/.gitignore | 1 + libminiooni/libminiooni.go | 101 ++++++++++++++++---------------- libminiooni/libminiooni_test.go | 11 ++++ 3 files changed, 63 insertions(+), 50 deletions(-) create mode 100644 libminiooni/.gitignore create mode 100644 libminiooni/libminiooni_test.go diff --git a/libminiooni/.gitignore b/libminiooni/.gitignore new file mode 100644 index 00000000..854297a1 --- /dev/null +++ b/libminiooni/.gitignore @@ -0,0 +1 @@ +/report.jsonl diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go index 8d844bd0..c8007e5e 100644 --- a/libminiooni/libminiooni.go +++ b/libminiooni/libminiooni.go @@ -22,19 +22,20 @@ import ( "github.com/pborman/getopt/v2" ) -type options struct { - annotations []string - bouncerURL string - collectorURL string - inputs []string - extraOptions []string - noBouncer bool - noGeoIP bool - noJSON bool - noCollector bool - proxy string - reportfile string - verbose bool +// Options contains the options you can set from the CLI. +type Options struct { + Annotations []string + BouncerURL string + CollectorURL string + Inputs []string + ExtraOptions []string + NoBouncer bool + NoGeoIP bool + NoJSON bool + NoCollector bool + Proxy string + ReportFile string + Verbose bool } const ( @@ -43,51 +44,51 @@ const ( ) var ( - globalOptions options + globalOptions Options startTime = time.Now() ) func init() { getopt.FlagLong( - &globalOptions.annotations, "annotation", 'A', "Add annotaton", "KEY=VALUE", + &globalOptions.Annotations, "annotation", 'A', "Add annotaton", "KEY=VALUE", ) getopt.FlagLong( - &globalOptions.bouncerURL, "bouncer", 'b', "Set bouncer base URL", "URL", + &globalOptions.BouncerURL, "bouncer", 'b', "Set bouncer base URL", "URL", ) getopt.FlagLong( - &globalOptions.collectorURL, "collector", 'c', + &globalOptions.CollectorURL, "collector", 'c', "Set collector base URL", "URL", ) getopt.FlagLong( - &globalOptions.inputs, "input", 'i', + &globalOptions.Inputs, "input", 'i', "Add test-dependent input to the test input", "INPUT", ) getopt.FlagLong( - &globalOptions.extraOptions, "option", 'O', + &globalOptions.ExtraOptions, "option", 'O', "Pass an option to the experiment", "KEY=VALUE", ) getopt.FlagLong( - &globalOptions.noBouncer, "no-bouncer", 0, "Don't use the OONI bouncer", + &globalOptions.NoBouncer, "no-bouncer", 0, "Don't use the OONI bouncer", ) getopt.FlagLong( - &globalOptions.noGeoIP, "no-geoip", 'g', + &globalOptions.NoGeoIP, "no-geoip", 'g', "Disable GeoIP lookup (not implemented!)", ) getopt.FlagLong( - &globalOptions.noJSON, "no-json", 'N', "Disable writing to disk", + &globalOptions.NoJSON, "no-json", 'N', "Disable writing to disk", ) getopt.FlagLong( - &globalOptions.noCollector, "no-collector", 'n', "Don't use a collector", + &globalOptions.NoCollector, "no-collector", 'n', "Don't use a collector", ) getopt.FlagLong( - &globalOptions.proxy, "proxy", 'P', "Set the proxy URL", "URL", + &globalOptions.Proxy, "proxy", 'P', "Set the proxy URL", "URL", ) getopt.FlagLong( - &globalOptions.reportfile, "reportfile", 'o', + &globalOptions.ReportFile, "reportfile", 'o', "Set the report file path", "PATH", ) getopt.FlagLong( - &globalOptions.verbose, "verbose", 'v', "Increase verbosity", + &globalOptions.Verbose, "verbose", 'v', "Increase verbosity", ) } @@ -189,16 +190,16 @@ func gethomedir() string { // // This function will panic in case of a fatal error. It is up to you that // integrate this function to either handle the panic of ignore it. -func MainWithConfiguration(experimentName string, currentOptions options) { - extraOptions := mustMakeMap(currentOptions.extraOptions) - annotations := mustMakeMap(currentOptions.annotations) +func MainWithConfiguration(experimentName string, currentOptions Options) { + extraOptions := mustMakeMap(currentOptions.ExtraOptions) + annotations := mustMakeMap(currentOptions.Annotations) logger := &log.Logger{Level: log.InfoLevel, Handler: &logHandler{Writer: os.Stderr}} - if currentOptions.verbose { + if currentOptions.Verbose { logger.Level = log.DebugLevel } - if currentOptions.reportfile == "" { - currentOptions.reportfile = "report.jsonl" + if currentOptions.ReportFile == "" { + currentOptions.ReportFile = "report.jsonl" } log.Log = logger @@ -214,8 +215,8 @@ func MainWithConfiguration(experimentName string, currentOptions options) { log.Debugf("miniooni temporary directory: %s", tempDir) var proxyURL *url.URL - if currentOptions.proxy != "" { - proxyURL = mustParseURL(currentOptions.proxy) + if currentOptions.Proxy != "" { + proxyURL = mustParseURL(currentOptions.Proxy) } kvstore2dir := filepath.Join(miniooniDir, "kvstore2") @@ -240,24 +241,24 @@ func MainWithConfiguration(experimentName string, currentOptions options) { ) }() - if currentOptions.bouncerURL != "" { - sess.AddAvailableHTTPSBouncer(currentOptions.bouncerURL) + if currentOptions.BouncerURL != "" { + sess.AddAvailableHTTPSBouncer(currentOptions.BouncerURL) } - if currentOptions.collectorURL != "" { + if currentOptions.CollectorURL != "" { // Implementation note: setting the collector before doing the lookup // is totally fine because it's a maybe lookup, meaning that any bit // of information already available will not be looked up again. - sess.AddAvailableHTTPSCollector(currentOptions.collectorURL) + sess.AddAvailableHTTPSCollector(currentOptions.CollectorURL) } - if !currentOptions.noBouncer { + if !currentOptions.NoBouncer { log.Info("Looking up OONI backends; please be patient...") err := sess.MaybeLookupBackends() fatalOnError(err, "cannot lookup OONI backends") } // See https://github.com/ooni/probe-engine/issues/297 fatalIfFalse( - currentOptions.noGeoIP == false, + currentOptions.NoGeoIP == false, "Sorry, the -g option is not implemented.", ) log.Info("Looking up your location; please be patient...") @@ -273,21 +274,21 @@ func MainWithConfiguration(experimentName string, currentOptions options) { builder, err := sess.NewExperimentBuilder(experimentName) fatalOnError(err, "cannot create experiment builder") if builder.NeedsInput() { - if len(currentOptions.inputs) <= 0 { + if len(currentOptions.Inputs) <= 0 { log.Info("Fetching test lists") list, err := sess.QueryTestListsURLs(&engine.TestListsURLsConfig{ Limit: 16, }) fatalOnError(err, "cannot fetch test lists") for _, entry := range list.Result { - currentOptions.inputs = append(currentOptions.inputs, entry.URL) + currentOptions.Inputs = append(currentOptions.Inputs, entry.URL) } } - } else if len(currentOptions.inputs) != 0 { + } else if len(currentOptions.Inputs) != 0 { fatalWithString("this experiment does not expect any input") } else { // Tests that do not expect input internally require an empty input to run - currentOptions.inputs = append(currentOptions.inputs, "") + currentOptions.Inputs = append(currentOptions.Inputs, "") } for key, value := range extraOptions { if value == "true" || value == "false" { @@ -306,7 +307,7 @@ func MainWithConfiguration(experimentName string, currentOptions options) { ) }() - if !currentOptions.noCollector { + if !currentOptions.NoCollector { log.Info("Opening report; please be patient...") err := experiment.OpenReport() fatalOnError(err, "cannot open report") @@ -314,9 +315,9 @@ func MainWithConfiguration(experimentName string, currentOptions options) { log.Infof("Report ID: %s", experiment.ReportID()) } - inputCount := len(currentOptions.inputs) + inputCount := len(currentOptions.Inputs) inputCounter := 0 - for _, input := range currentOptions.inputs { + for _, input := range currentOptions.Inputs { inputCounter++ if input != "" { log.Infof("[%d/%d] running with input: %s", inputCounter, inputCount, input) @@ -324,16 +325,16 @@ func MainWithConfiguration(experimentName string, currentOptions options) { measurement, err := experiment.Measure(input) warnOnError(err, "measurement failed") measurement.AddAnnotations(annotations) - if !currentOptions.noCollector { + if !currentOptions.NoCollector { log.Infof("submitting measurement to OONI collector; please be patient...") err := experiment.SubmitAndUpdateMeasurement(measurement) warnOnError(err, "submitting measurement failed") } - if !currentOptions.noJSON { + if !currentOptions.NoJSON { // Note: must be after submission because submission modifies // the measurement to include the report ID. log.Infof("saving measurement to disk") - err := experiment.SaveMeasurement(measurement, currentOptions.reportfile) + err := experiment.SaveMeasurement(measurement, currentOptions.ReportFile) warnOnError(err, "saving measurement failed") } } diff --git a/libminiooni/libminiooni_test.go b/libminiooni/libminiooni_test.go new file mode 100644 index 00000000..cfcbb025 --- /dev/null +++ b/libminiooni/libminiooni_test.go @@ -0,0 +1,11 @@ +package libminiooni_test + +import ( + "testing" + + "github.com/ooni/probe-engine/libminiooni" +) + +func TestIntegrationSimple(t *testing.T) { + libminiooni.MainWithConfiguration("example", libminiooni.Options{}) +} From 12e0ab8d1335ead20fcf25c6e55974bc2ca38837 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Mon, 4 May 2020 12:28:17 +0200 Subject: [PATCH 9/9] libminiooni: set the correct version number --- libminiooni/libminiooni.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libminiooni/libminiooni.go b/libminiooni/libminiooni.go index c8007e5e..6868acec 100644 --- a/libminiooni/libminiooni.go +++ b/libminiooni/libminiooni.go @@ -19,6 +19,7 @@ import ( "github.com/apex/log" engine "github.com/ooni/probe-engine" "github.com/ooni/probe-engine/internal/humanizex" + "github.com/ooni/probe-engine/version" "github.com/pborman/getopt/v2" ) @@ -40,7 +41,7 @@ type Options struct { const ( softwareName = "miniooni" - softwareVersion = "0.1.0-dev" + softwareVersion = version.Version ) var (