Merge da6d877 into f7f4a43

cmd/miniooni/main.go

@@ -224,15 +224,14 @@ func main() {
 	if builder.NeedsInput() {
 		if len(globalOptions.inputs) <= 0 {
 			log.Info("Fetching test lists")
-			config := sess.NewTestListsConfig()
-			config.Limit = 16
-			client := sess.NewTestListsClient()
-			list, err := client.Fetch(config)
+			list, err := sess.QueryTestListsURLs(&engine.TestListsURLsConfig{
+				Limit: 16,
+			})
 			if err != nil {
 				log.WithError(err).Fatal("cannot fetch test lists")
 			}
-			for _, entry := range list {
-				globalOptions.inputs = append(globalOptions.inputs, entry.URL())
+			for _, entry := range list.Result {
+				globalOptions.inputs = append(globalOptions.inputs, entry.URL)
 			}
 		}
 	} else if len(globalOptions.inputs) != 0 {

internal/orchestra/testlists/urls/urls.go

@@ -0,0 +1,51 @@
+// Package urls queries orchestra test-lists/urls API
+package urls
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/ooni/probe-engine/httpx/jsonapi"
+	"github.com/ooni/probe-engine/log"
+	"github.com/ooni/probe-engine/model"
+)
+
+// Config contains configs for querying tests-lists/urls
+type Config struct {
+	BaseURL           string
+	CountryCode       string
+	EnabledCategories []string
+	HTTPClient        *http.Client
+	Limit             int
+	Logger            log.Logger
+	UserAgent         string
+}
+
+// Result contains the result returned by tests-lists/urls
+type Result struct {
+	Results []model.URLInfo `json:"results"`
+}
+
+// Query retrieves the test list for the specified country.
+func Query(ctx context.Context, config Config) (response Result, err error) {
+	query := url.Values{}
+	if config.CountryCode != "" {
+		query.Set("probe_cc", config.CountryCode)
+	}
+	if config.Limit > 0 {
+		query.Set("limit", fmt.Sprintf("%d", config.Limit))
+	}
+	if len(config.EnabledCategories) > 0 {
+		query.Set("category_codes", strings.Join(config.EnabledCategories, ","))
+	}
+	err = (&jsonapi.Client{
+		BaseURL:    config.BaseURL,
+		HTTPClient: config.HTTPClient,
+		Logger:     config.Logger,
+		UserAgent:  config.UserAgent,
+	}).ReadWithQuery(ctx, "/api/v1/test-list/urls", query, &response)
+	return
+}

internal/orchestra/testlists/urls/urls_test.go

@@ -0,0 +1,29 @@
+package urls
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/apex/log"
+)
+
+func TestIntegration(t *testing.T) {
+	config := Config{
+		BaseURL:           "https://orchestrate.ooni.io",
+		CountryCode:       "IT",
+		EnabledCategories: []string{"NEWS", "CULTR"},
+		HTTPClient:        http.DefaultClient,
+		Limit:             17,
+		Logger:            log.Log,
+		UserAgent:         "ooniprobe-engine/v0.1.0-dev",
+	}
+	ctx := context.Background()
+	result, err := Query(ctx, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(result.Results) < 1 {
+		t.Fatal("no results")
+	}
+}

model/model.go

@@ -215,3 +215,10 @@ const (
 
 // DefaultProbeASNString is the default probe ASN as a string.
 var DefaultProbeASNString = fmt.Sprintf("AS%d", DefaultProbeASN)
+
+// URLInfo contains info on a test lists URL
+type URLInfo struct {
+	CategoryCode string `json:"category_code"`
+	CountryCode  string `json:"country_code"`
+	URL          string `json:"url"`
+}

session.go

@@ -7,7 +7,6 @@ import (
 	"net/url"
 
 	"github.com/ooni/probe-engine/log"
-	"github.com/ooni/probe-engine/orchestra/testlists"
 	"github.com/ooni/probe-engine/session"
 )
 
@@ -124,22 +123,6 @@ func (sess *Session) SetIncludeProbeIP(value bool) {
 	sess.session.PrivacySettings.IncludeIP = value
 }
 
-// NewTestListsConfig returns prefilled settings for TestListsClient
-// where in particular we have set the correct country code
-func (sess *Session) NewTestListsConfig() *TestListsConfig {
-	return &TestListsConfig{
-		CountryCode: sess.session.ProbeCC(),
-	}
-}
-
-// NewTestListsClient returns a new TestListsClient that is configured
-// to perform requests in the context of this session
-func (sess *Session) NewTestListsClient() *TestListsClient {
-	return &TestListsClient{
-		client: testlists.NewClient(sess.session),
-	}
-}
-
 // NewExperimentBuilder returns a new experiment builder
 // for the experiment with the given name, or an error if
 // there's no such experiment with the given name

testlists.go

@@ -2,58 +2,66 @@ package engine
 
 import (
 	"context"
+	"errors"
 
-	"github.com/ooni/probe-engine/orchestra/testlists"
+	"github.com/ooni/probe-engine/internal/orchestra/testlists/urls"
+	"github.com/ooni/probe-engine/model"
 )
 
-// TestListsConfig contains settings for TestListsClient
-type TestListsConfig struct {
-	BaseURL             string   // or use default
-	AvailableCategories []string // or ask for all
-	CountryCode         string   // set by session
-	Limit               int      // or get all
+// TestListsURLsConfig config config for test-lists/urls API.
+type TestListsURLsConfig struct {
+	BaseURL    string   // URL to use (empty means default)
+	Categories []string // Categories to query for (empty means all)
+	Limit      int      // Max number of URLs (<= 0 means no limit)
 }
 
-// TestListsClient is a test lists client
-type TestListsClient struct {
-	client *testlists.Client
+// AddCategory adds a category to the list of categories to query. Not
+// adding any categories will query for URLs in all categories.
+func (c *TestListsURLsConfig) AddCategory(s string) {
+	c.Categories = append(c.Categories, s)
 }
 
-// URLInfo contains info about URLs
-type URLInfo interface {
-	URL() string
-	CategoryCode() string
-	CountryCode() string
+// TestListsURLsResult contains the results of calling the
+// test-lists/urls OONI orchestra API.
+type TestListsURLsResult struct {
+	Result []model.URLInfo
 }
 
-// Fetch fetches the test list
-func (c *TestListsClient) Fetch(config *TestListsConfig) ([]URLInfo, error) {
-	var out []URLInfo
-	if config.BaseURL != "" {
-		c.client.BaseURL = config.BaseURL
-	}
-	list, err := c.client.Do(context.Background(), config.CountryCode, config.Limit)
-	if err != nil {
-		return nil, err
-	}
-	for _, entry := range list {
-		out = append(out, &urlinfo{u: entry})
-	}
-	return out, nil
-}
-
-type urlinfo struct {
-	u testlists.URLInfo
+// Count returns the number of returned URLs
+func (r *TestListsURLsResult) Count() int64 {
+	return int64(len(r.Result))
 }
 
-func (u *urlinfo) URL() string {
-	return u.u.URL
-}
-
-func (u *urlinfo) CategoryCode() string {
-	return u.u.CategoryCode
+// At returns the URL at the given index or nil
+func (r *TestListsURLsResult) At(idx int64) (out model.URLInfo) {
+	if idx >= 0 && idx < int64(len(r.Result)) {
+		out = r.Result[int(idx)]
+	}
+	return
 }
 
-func (u *urlinfo) CountryCode() string {
-	return u.u.CountryCode
+// QueryTestListsURLs queries the test-lists/urls API.
+func (sess *Session) QueryTestListsURLs(
+	conf *TestListsURLsConfig,
+) (*TestListsURLsResult, error) {
+	if conf == nil {
+		return nil, errors.New("QueryTestListURLs: passed nil config")
+	}
+	baseURL := "https://orchestrate.ooni.io"
+	if conf.BaseURL != "" {
+		baseURL = conf.BaseURL
+	}
+	result, err := urls.Query(context.Background(), urls.Config{
+		BaseURL:           baseURL,
+		CountryCode:       sess.ProbeCC(),
+		EnabledCategories: conf.Categories,
+		HTTPClient:        sess.session.HTTPDefaultClient,
+		Limit:             conf.Limit,
+		Logger:            sess.session.Logger,
+		UserAgent:         sess.session.UserAgent(),
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &TestListsURLsResult{Result: result.Results}, nil
 }