Hostname iterator

api/client.go

@@ -316,20 +316,9 @@ func (cli *Client) Scan(url string, options ...ScanOption) (*ScanResult, error)
 	return r, nil
 }
 
-func (c *Client) GetHostname(hostname string, opts ...HostnameOption) (*Response, error) {
-	hostnameOpts := newHostnameOptions(opts...)
-
-	url := URL("/api/v1/hostname/%s", hostname)
-
-	// set query parameters
-	q := url.Query()
-	q.Add("limit", strconv.Itoa(hostnameOpts.Limit))
-	if hostnameOpts.PageState != "" {
-		q.Add("pageState", hostnameOpts.PageState)
-	}
-	url.RawQuery = q.Encode()
-
-	return c.Get(url)
+func (c *Client) IterateHostname(hostname string, opts ...HostnameIteratorOption) (*HostnameIterator, error) {
+	u := URL("/api/v1/hostname/%s", hostname)
+	return newHostnameIterator(c, u, opts...)
 }
 
 func (cli *Client) GetResult(uuid string) (*Response, error) {

api/hostname_iterator.go

@@ -0,0 +1,148 @@
+package api
+
+import (
+	"encoding/json"
+	"iter"
+	"net/url"
+	"strconv"
+)
+
+type HostnameResults struct {
+	Item      string            `json:"item"`
+	Results   []json.RawMessage `json:"results"`
+	PageState string            `json:"pageState"`
+	Raw       json.RawMessage   `json:"-"`
+}
+
+func (r *HostnameResults) UnmarshalJSON(data []byte) error {
+	type results HostnameResults
+	var dst results
+
+	err := json.Unmarshal(data, &dst)
+	if err != nil {
+		return err
+	}
+	*r = HostnameResults(dst)
+	r.Raw = data
+	return err
+}
+
+type HostnameIteratorOption func(*HostnameIterator) error
+
+// size is the number of results returned by the iterator in each batch.
+func HostnameIteratorSize(size int) HostnameIteratorOption {
+	return func(it *HostnameIterator) error {
+		it.size = size
+		return nil
+	}
+}
+
+// limit is the maximum number of results that will be returned by the iterator.
+// note that this is not the same as the API endpoint's "limit" query parameter.
+func HostnameIteratorLimit(limit int) HostnameIteratorOption {
+	return func(it *HostnameIterator) error {
+		it.limit = limit
+		return nil
+	}
+}
+
+func HostnameIteratorPageState(pageState string) HostnameIteratorOption {
+	return func(it *HostnameIterator) error {
+		it.PageState = pageState
+		return nil
+	}
+}
+
+type HostnameIterator struct {
+	client    *Client
+	limit     int
+	size      int
+	count     int
+	PageState string
+	link      *url.URL
+	HasMore   bool
+}
+
+func newHostnameIterator(cli *Client, u *url.URL, options ...HostnameIteratorOption) (*HostnameIterator, error) {
+	it := &HostnameIterator{
+		client:  cli,
+		HasMore: true,
+		count:   0,
+	}
+
+	for _, opt := range options {
+		if err := opt(it); err != nil {
+			return nil, err
+		}
+	}
+
+	query := u.Query()
+
+	// size (number of results per batch) is "limit" in this API endpoint
+	if it.size > 0 {
+		query.Add("limit", strconv.Itoa(it.size))
+	}
+
+	if it.PageState != "" {
+		query.Add("pageState", it.PageState)
+	}
+
+	u.RawQuery = query.Encode()
+	it.link = u
+
+	return it, nil
+}
+
+func (it *HostnameIterator) getMoreResults() (results []*json.RawMessage, err error) {
+	resp, err := it.client.Get(it.link)
+	if err != nil {
+		return nil, err
+	}
+
+	r := &HostnameResults{}
+	err = json.Unmarshal(resp.Raw, r)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, result := range r.Results {
+		results = append(results, &result)
+	}
+
+	// update pageState for the next request
+	q := it.link.Query()
+	q.Set("pageState", r.PageState)
+	it.link.RawQuery = q.Encode()
+
+	// update HasMore based on the number of results
+	it.HasMore = len(r.Results) >= it.size
+
+	return results, nil
+}
+
+func (it *HostnameIterator) Iterate() iter.Seq2[*json.RawMessage, error] {
+	return func(yield func(*json.RawMessage, error) bool) {
+		for it.count < it.limit {
+			results, err := it.getMoreResults()
+			if err != nil {
+				yield(nil, err)
+				return
+			}
+
+			for _, result := range results {
+				if !yield(result, nil) {
+					return
+				}
+
+				it.count++
+				if it.count >= it.limit {
+					return
+				}
+			}
+
+			if len(results) == 0 || !it.HasMore {
+				return
+			}
+		}
+	}
+}

cmd/pro/hostname.go

@@ -0,0 +1,93 @@
+package pro
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"github.com/urlscan/urlscan-cli/api"
+	"github.com/urlscan/urlscan-cli/pkg/utils"
+)
+
+type HostnameResults struct {
+	Results   []json.RawMessage `json:"results"`
+	PageState string            `json:"pageState"`
+	HasMore   bool              `json:"has_more"`
+}
+
+func newHostnameResults() HostnameResults {
+	return HostnameResults{
+		Results:   make([]json.RawMessage, 0),
+		PageState: "",
+	}
+}
+
+var hostnameCmdExample = `  urlscan pro hostname <hostname>
+  echo "<hostname>" | urlscan pro hostname -`
+
+var hostnameLong = `To have the same idiom with the search command, this command has the following specs:
+
+- Request:
+  - limit: the maximum number of results that will be returned by the iterator.
+  - size: the number of results returned by the iterator in each batch (equivalent to the API endpoint's "limit" query parameter).
+- Response:
+  - hasMore: indicates more results are available.`
+
+var hostnameCmd = &cobra.Command{
+	Use:     "hostname",
+	Short:   "Get the historical observations for a specific hostname in the hostname data source",
+	Long:    hostnameLong,
+	Example: hostnameCmdExample,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		size, _ := cmd.Flags().GetInt("size")
+		limit, _ := cmd.Flags().GetInt("limit")
+		pageState, _ := cmd.Flags().GetString("page-state")
+
+		reader := utils.StringReaderFromCmdArgs(args)
+		hostname, err := reader.ReadString()
+		if err != nil {
+			return err
+		}
+
+		client, err := utils.NewAPIClient()
+		if err != nil {
+			return err
+		}
+
+		it, err := client.IterateHostname(hostname,
+			api.HostnameIteratorLimit(limit),
+			api.HostnameIteratorSize(size),
+			api.HostnameIteratorPageState(pageState),
+		)
+		if err != nil {
+			return err
+		}
+
+		results := newHostnameResults()
+		for result, err := range it.Iterate() {
+			if err != nil {
+				return err
+			}
+			results.Results = append(results.Results, *result)
+		}
+
+		results.HasMore = it.HasMore
+
+		b, err := json.Marshal(results)
+		if err != nil {
+			return err
+		}
+
+		fmt.Print(string(b))
+
+		return nil
+	},
+}
+
+func init() {
+	hostnameCmd.Flags().IntP("limit", "l", 10000, "Maximum number of results that will be returned by the iterator")
+	hostnameCmd.Flags().IntP("size", "s", 1000, "Number of results returned by the iterator in each batch")
+	hostnameCmd.Flags().StringP("page-state", "p", "", "Returns additional results starting from this page state from the previous API call")
+
+	RootCmd.AddCommand(hostnameCmd)
+}

docs/urlscan.md

@@ -10,7 +10,6 @@ A CLI tool for interacting with urlscan.io
 
 ### SEE ALSO
 
-* [urlscan hostname](urlscan_hostname.md)	 - Get the historical observations for a specific hostname in the hostname data source
 * [urlscan key](urlscan_key.md)	 - Manage API key
 * [urlscan pro](urlscan_pro.md)	 - Pro sub-commands
 * [urlscan quotas](urlscan_quotas.md)	 - Get API quotas

docs/urlscan_pro.md

@@ -12,6 +12,7 @@ Pro sub-commands
 
 * [urlscan](urlscan.md)	 - A CLI tool for interacting with urlscan.io
 * [urlscan pro brand](urlscan_pro_brand.md)	 - Brand sub-commands
+* [urlscan pro hostname](urlscan_pro_hostname.md)	 - Get the historical observations for a specific hostname in the hostname data source
 * [urlscan pro incident](urlscan_pro_incident.md)	 - Incident sub-commands
 * [urlscan pro saved-search](urlscan_pro_saved-search.md)	 - Saved search sub-commands
 * [urlscan pro structure-search](urlscan_pro_structure-search.md)	 - Get structurally similar results to a specific scan