From 920e42987280af9084e4cbc19d19e1482218cc25 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 5 May 2026 11:31:32 +1200 Subject: [PATCH 1/6] cp the zannotate binary rather than mv to speed up compile times --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f717911..58f249a 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ zannotate: cd cmd/zannotate && \ go build -o zannotate && \ cd - && \ - mv cmd/zannotate/zannotate . + cp cmd/zannotate/zannotate . clean: rm -f zannotate From b65967100eae04cdd920a0db269dac64658f8f90 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 5 May 2026 11:38:17 +1200 Subject: [PATCH 2/6] group CLI flags by annotation module for an improved --help UX --- censys.go | 2 ++ cmd/zannotate/main.go | 53 +++++++++++++++++++++++++++++++++++++++---- conf.go | 1 + geoip.go | 2 ++ geoipasn.go | 2 ++ greynoise_psychic.go | 2 ++ ipinfo.go | 2 ++ rdap.go | 2 ++ rdns.go | 2 ++ routing.go | 2 ++ spur.go | 2 ++ 11 files changed, 68 insertions(+), 4 deletions(-) diff --git a/censys.go b/censys.go index c7de837..aa3c389 100644 --- a/censys.go +++ b/censys.go @@ -56,6 +56,8 @@ func (a *CensysAnnotatorFactory) IsEnabled() bool { return a.Enabled } +func (a *CensysAnnotatorFactory) GroupName() string { return "Censys" } + func (a *CensysAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&a.Enabled, "censys", false, "censys internet intelligence") flags.StringVar(&a.personalToken, "censys-pat", "", "censys API personal access token (PAT)") diff --git a/cmd/zannotate/main.go b/cmd/zannotate/main.go index d89fa61..d4f11d4 100644 --- a/cmd/zannotate/main.go +++ b/cmd/zannotate/main.go @@ -16,6 +16,8 @@ package main import ( "flag" + "fmt" + "maps" "os" "slices" @@ -41,10 +43,7 @@ func main() { // encode/decode threads flags.IntVar(&conf.InputDecodeThreads, "input-decode-threads", 3, "number of golang processes to decode input data (e.g., json)") flags.IntVar(&conf.OutputEncodeThreads, "output-encode-threads", 3, "number of golang processes to encode output data (e.g., json)") - // add the flags defined by each of the annotation modules - for _, annotator := range zannotate.Annotators { - annotator.AddFlags(flags) - } + prepareUsageString(flags) // Constructs the --help string // parse err := flags.Parse(os.Args[1:]) if err != nil { @@ -94,3 +93,49 @@ func main() { // perform annotation zannotate.DoAnnotation(&conf) } + +func prepareUsageString(flags *flag.FlagSet) { + // Build grouped flag help: snapshot flags before each annotator registers its own. + type flagGroup struct { + name string + flags []string + } + snapshot := func() map[string]bool { + seen := make(map[string]bool) + flags.VisitAll(func(f *flag.Flag) { seen[f.Name] = true }) + return seen + } + groups := make([]flagGroup, 0, 1 + len(zannotate.Annotators)) // Global options + each annotator + // Collect the global flags registered above. + globalFlags := snapshot() + globalNames := slices.Collect(maps.Keys(globalFlags)) + groups = append(groups, flagGroup{"Global", globalNames}) + // add the flags defined by each of the annotation modules + for _, annotator := range zannotate.Annotators { + pre := snapshot() + annotator.AddFlags(flags) + var added []string + flags.VisitAll(func(f *flag.Flag) { + if !pre[f.Name] { + added = append(added, f.Name) + } + }) + if len(added) > 0 { + groups = append(groups, flagGroup{annotator.GroupName(), added}) + } + } + flags.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [options] <-module [module-options]>...\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "At least one annotation module must be specified (e.g. -geoip2, -rdns, -routing).\n\n") + for _, g := range groups { + fmt.Fprintf(os.Stderr, "\n%s Options:\n", g.name) + // Create a temporary FlagSet containing only this group's flags so we + // can delegate formatting to PrintDefaults. + tmp := flag.NewFlagSet("", flag.ContinueOnError) + for _, name := range g.flags { + tmp.Var(flags.Lookup(name).Value, name, flags.Lookup(name).Usage) + } + tmp.PrintDefaults() + } + } +} diff --git a/conf.go b/conf.go index 8f158bc..c538ee8 100644 --- a/conf.go +++ b/conf.go @@ -31,6 +31,7 @@ type Annotator interface { type AnnotatorFactory interface { Initialize(c *GlobalConf) error AddFlags(flags *flag.FlagSet) + GroupName() string GetWorkers() int IsEnabled() bool MakeAnnotator(i int) Annotator diff --git a/geoip.go b/geoip.go index 9d67241..a0e7840 100644 --- a/geoip.go +++ b/geoip.go @@ -92,6 +92,8 @@ type GeoIP2Annotator struct { } // GeoIP2 Annotator Factory (Global) +func (a *GeoIP2AnnotatorFactory) GroupName() string { return "GeoIP2" } + func (a *GeoIP2AnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&a.Enabled, "geoip2", false, "annotate with Maxmind GeoIP2/GeoLite data") flags.StringVar(&a.Path, "geoip2-database", "", diff --git a/geoipasn.go b/geoipasn.go index 3b7c81b..dc1b7ac 100644 --- a/geoipasn.go +++ b/geoipasn.go @@ -41,6 +41,8 @@ type GeoIPASNAnnotator struct { Id int } +func (fact *GeoIPASNAnnotatorFactory) GroupName() string { return "GeoASN" } + func (fact *GeoIPASNAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&fact.Enabled, "geoasn", false, "annotate with Maxmind GeoLite/GeoIP ASN data") flags.StringVar(&fact.Path, "geoasn-database", "", "path to Maxmind ASN database") diff --git a/greynoise_psychic.go b/greynoise_psychic.go index 71faa39..15775fd 100644 --- a/greynoise_psychic.go +++ b/greynoise_psychic.go @@ -71,6 +71,8 @@ func (a *GreyNoiseAnnotatorFactory) IsEnabled() bool { return a.Enabled } +func (a *GreyNoiseAnnotatorFactory) GroupName() string { return "GreyNoise" } + func (a *GreyNoiseAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&a.Enabled, "greynoise", false, "greynoise psychic data intelligence") flags.StringVar(&a.DBPath, "greynoise-database", "", "path to greynoise psychic .mmdb file") diff --git a/ipinfo.go b/ipinfo.go index 88368fb..e071065 100644 --- a/ipinfo.go +++ b/ipinfo.go @@ -236,6 +236,8 @@ func (a *IPInfoAnnotatorFactory) IsEnabled() bool { return a.Enabled } +func (a *IPInfoAnnotatorFactory) GroupName() string { return "IPInfo" } + func (a *IPInfoAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&a.Enabled, "ipinfo", false, "annotate with IPInfo.io data using a local MaxMind DB file") flags.StringVar(&a.DatabaseFilePath, "ipinfo-database", "", "path to MaxMind DB data file for IPInfo.io annotation") diff --git a/rdap.go b/rdap.go index e52a6dd..46500ed 100644 --- a/rdap.go +++ b/rdap.go @@ -35,6 +35,8 @@ type RDAPAnnotator struct { } // RDAP Annotator Factory (Global) +func (a *RDAPAnnotatorFactory) GroupName() string { return "RDAP" } + func (a *RDAPAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&a.Enabled, "rdap", false, "annotate with RDAP (successor to WHOIS) lookup") flags.IntVar(&a.Threads, "rdap-threads", 5, "how many rdap processing threads to use") diff --git a/rdns.go b/rdns.go index 5003938..d2b3625 100644 --- a/rdns.go +++ b/rdns.go @@ -46,6 +46,8 @@ type RDNSAnnotator struct { // RDNS Annotator Factory (Global) +func (a *RDNSAnnotatorFactory) GroupName() string { return "Reverse DNS" } + func (a *RDNSAnnotatorFactory) MakeAnnotator(i int) Annotator { var v RDNSAnnotator v.Factory = a diff --git a/routing.go b/routing.go index 56cf06a..48f129f 100644 --- a/routing.go +++ b/routing.go @@ -42,6 +42,8 @@ type RoutingAnnotator struct { } // Routing Annotator Factory (Global) +func (a *RoutingAnnotatorFactory) GroupName() string { return "Routing/BGP" } + func (a *RoutingAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&a.Enabled, "routing", false, "annotate with origin AS lookup") flags.StringVar(&a.RoutingTablePath, "routing-mrt-file", "", diff --git a/spur.go b/spur.go index f977734..f074aca 100644 --- a/spur.go +++ b/spur.go @@ -74,6 +74,8 @@ func (a *SpurAnnotatorFactory) IsEnabled() bool { return a.Enabled } +func (a *SpurAnnotatorFactory) GroupName() string { return "Spur" } + func (a *SpurAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&a.Enabled, "spur", false, "enrich with Spur's threat intelligence data") flags.IntVar(&a.Threads, "spur-threads", 100, "how many threads to use for Spur lookups") From 325407dace0bf11914de6fca48605c35123ac5f6 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 5 May 2026 11:55:59 +1200 Subject: [PATCH 3/6] specify optional global options --- cmd/zannotate/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zannotate/main.go b/cmd/zannotate/main.go index d4f11d4..2d8ceb6 100644 --- a/cmd/zannotate/main.go +++ b/cmd/zannotate/main.go @@ -125,7 +125,7 @@ func prepareUsageString(flags *flag.FlagSet) { } } flags.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s [options] <-module [module-options]>...\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [global options] <-module [module-options]>...\n", os.Args[0]) fmt.Fprintf(os.Stderr, "At least one annotation module must be specified (e.g. -geoip2, -rdns, -routing).\n\n") for _, g := range groups { fmt.Fprintf(os.Stderr, "\n%s Options:\n", g.name) From 1434e85b5fd0d8724fef02f7c62fddb79dda0644 Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 5 May 2026 11:59:46 +1200 Subject: [PATCH 4/6] clean up global flags and remove unused metadata flag --- cmd/zannotate/main.go | 6 ++---- conf.go | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/zannotate/main.go b/cmd/zannotate/main.go index 2d8ceb6..8a92e7b 100644 --- a/cmd/zannotate/main.go +++ b/cmd/zannotate/main.go @@ -30,11 +30,9 @@ func main() { var conf zannotate.GlobalConf flags := flag.NewFlagSet("flags", flag.ExitOnError) - flags.StringVar(&conf.InputFilePath, "input-file", "-", "ip addresses to read") + flags.StringVar(&conf.InputFilePath, "input-file", "-", "ip addresses to read, use '-' for std in") flags.StringVar(&conf.InputFileType, "input-file-type", "ips", "ips, csv, json") - flags.StringVar(&conf.OutputFilePath, "output-file", "-", "where should JSON output be saved") - flags.StringVar(&conf.MetadataFilePath, "metadata-file", "", - "where should JSON metadata be saved") + flags.StringVar(&conf.OutputFilePath, "output-file", "-", "where should JSON output be saved, use '-' for stdout") flags.StringVar(&conf.LogFilePath, "log-file", "", "where should JSON logs be saved") flags.IntVar(&conf.Verbosity, "verbosity", 3, "log verbosity: 1 (lowest)--5 (highest)") // json annotation configuration diff --git a/conf.go b/conf.go index c538ee8..3cf5e2d 100644 --- a/conf.go +++ b/conf.go @@ -49,7 +49,6 @@ type GlobalConf struct { InputFilePath string InputFileType string OutputFilePath string - MetadataFilePath string LogFilePath string Verbosity int Threads int From 6aaebaa105e1d4110f77b2ee775c07f21ed4412c Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 5 May 2026 12:06:05 +1200 Subject: [PATCH 5/6] clean up module flags --- censys.go | 2 +- geoip.go | 4 ++-- geoipasn.go | 2 +- greynoise_psychic.go | 2 +- ipinfo.go | 2 +- rdap.go | 2 +- rdns.go | 4 ++-- routing.go | 2 +- spur.go | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/censys.go b/censys.go index aa3c389..bb98061 100644 --- a/censys.go +++ b/censys.go @@ -59,7 +59,7 @@ func (a *CensysAnnotatorFactory) IsEnabled() bool { func (a *CensysAnnotatorFactory) GroupName() string { return "Censys" } func (a *CensysAnnotatorFactory) AddFlags(flags *flag.FlagSet) { - flags.BoolVar(&a.Enabled, "censys", false, "censys internet intelligence") + flags.BoolVar(&a.Enabled, "censys", false, "annotate with censys internet intelligence") flags.StringVar(&a.personalToken, "censys-pat", "", "censys API personal access token (PAT)") flags.IntVar(&a.Threads, "censys-threads", 1, "how many enrichment threads to use. Note that free plan only allows 1 concurrent API request at a time") } diff --git a/geoip.go b/geoip.go index a0e7840..c35a97d 100644 --- a/geoip.go +++ b/geoip.go @@ -103,8 +103,8 @@ func (a *GeoIP2AnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.StringVar(&a.Language, "geoip2-language", "en", "what language geoip2 database is in") flags.StringVar(&a.RawInclude, "geoip2-fields", "*", - "city, continent, country, location, postal, registered_country, subdivisions, traits") - flags.IntVar(&a.Threads, "geoip2-threads", 5, "how many geoIP processing threads to use") + "comma-separated list of geoip annotations, '*' for all: city, continent, country, location, postal, registered_country, subdivisions, traits") + flags.IntVar(&a.Threads, "geoip2-threads", 5, "how many processing threads to use") } func (a *GeoIP2AnnotatorFactory) IsEnabled() bool { diff --git a/geoipasn.go b/geoipasn.go index dc1b7ac..02dc423 100644 --- a/geoipasn.go +++ b/geoipasn.go @@ -47,7 +47,7 @@ func (fact *GeoIPASNAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&fact.Enabled, "geoasn", false, "annotate with Maxmind GeoLite/GeoIP ASN data") flags.StringVar(&fact.Path, "geoasn-database", "", "path to Maxmind ASN database") flags.StringVar(&fact.Mode, "geoasn-mode", "mmap", "how to open database: 'mmap' or 'memory'") - flags.IntVar(&fact.Threads, "geoasn-threads", 5, "how many geoASN processing threads to use") + flags.IntVar(&fact.Threads, "geoasn-threads", 5, "how many processing threads to use") } func (fact *GeoIPASNAnnotatorFactory) IsEnabled() bool { diff --git a/greynoise_psychic.go b/greynoise_psychic.go index 15775fd..d657a73 100644 --- a/greynoise_psychic.go +++ b/greynoise_psychic.go @@ -74,7 +74,7 @@ func (a *GreyNoiseAnnotatorFactory) IsEnabled() bool { func (a *GreyNoiseAnnotatorFactory) GroupName() string { return "GreyNoise" } func (a *GreyNoiseAnnotatorFactory) AddFlags(flags *flag.FlagSet) { - flags.BoolVar(&a.Enabled, "greynoise", false, "greynoise psychic data intelligence") + flags.BoolVar(&a.Enabled, "greynoise", false, "annotate with greynoise psychic data intelligence") flags.StringVar(&a.DBPath, "greynoise-database", "", "path to greynoise psychic .mmdb file") flags.IntVar(&a.Threads, "greynoise-threads", 2, "how many enrichment threads to use") } diff --git a/ipinfo.go b/ipinfo.go index e071065..af7253d 100644 --- a/ipinfo.go +++ b/ipinfo.go @@ -242,7 +242,7 @@ func (a *IPInfoAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&a.Enabled, "ipinfo", false, "annotate with IPInfo.io data using a local MaxMind DB file") flags.StringVar(&a.DatabaseFilePath, "ipinfo-database", "", "path to MaxMind DB data file for IPInfo.io annotation") // On a quick benchmark of 1M IPs using a local DB file on a M2 Macbook Air, 1 thread vs. 10 threads were about the same speed, annotating about 212k IPs/second. - flags.IntVar(&a.Threads, "ipinfo-threads", 1, "how many ipinfo annotator threads") + flags.IntVar(&a.Threads, "ipinfo-threads", 1, "how many annotator threads") } // IPInfo Annotator (Per-Worker) diff --git a/rdap.go b/rdap.go index 46500ed..871e1db 100644 --- a/rdap.go +++ b/rdap.go @@ -39,7 +39,7 @@ func (a *RDAPAnnotatorFactory) GroupName() string { return "RDAP" } func (a *RDAPAnnotatorFactory) AddFlags(flags *flag.FlagSet) { flags.BoolVar(&a.Enabled, "rdap", false, "annotate with RDAP (successor to WHOIS) lookup") - flags.IntVar(&a.Threads, "rdap-threads", 5, "how many rdap processing threads to use") + flags.IntVar(&a.Threads, "rdap-threads", 5, "how many processing threads to use") flags.IntVar(&a.Timeout, "rdap-timeout", 5, "RDAP query timeout in seconds") } diff --git a/rdns.go b/rdns.go index d2b3625..551a97c 100644 --- a/rdns.go +++ b/rdns.go @@ -101,9 +101,9 @@ func (a *RDNSAnnotatorFactory) IsEnabled() bool { func (a *RDNSAnnotatorFactory) AddFlags(flags *flag.FlagSet) { // Reverse DNS Lookup - flags.BoolVar(&a.Enabled, "rdns", false, "reverse dns lookup") + flags.BoolVar(&a.Enabled, "rdns", false, "annotate with reverse dns lookup") flags.StringVar(&a.RawResolvers, "rdns-dns-servers", "", "list of DNS servers to use for DNS lookups, comma-separated IP list. If empty, will use system defaults") - flags.IntVar(&a.Threads, "rdns-threads", 100, "how many reverse dns threads") + flags.IntVar(&a.Threads, "rdns-threads", 100, "how many annotation threads to use") flags.IntVar(&a.timeoutSecs, "rdns-timeout", 2, "timeout for each rdns query, in seconds") } diff --git a/routing.go b/routing.go index 48f129f..1b6edf0 100644 --- a/routing.go +++ b/routing.go @@ -50,7 +50,7 @@ func (a *RoutingAnnotatorFactory) AddFlags(flags *flag.FlagSet) { "path to MRT TABLE_DUMPv2 file") flags.StringVar(&a.ASNamesPath, "routing-as-names", "", "path to as names file") flags.IntVar(&a.Threads, "routing-threads", 5, - "how many routing processing threads to use") + "how many processing threads to use") } func (a *RoutingAnnotatorFactory) IsEnabled() bool { diff --git a/spur.go b/spur.go index f074aca..6ba6228 100644 --- a/spur.go +++ b/spur.go @@ -77,7 +77,7 @@ func (a *SpurAnnotatorFactory) IsEnabled() bool { func (a *SpurAnnotatorFactory) GroupName() string { return "Spur" } func (a *SpurAnnotatorFactory) AddFlags(flags *flag.FlagSet) { - flags.BoolVar(&a.Enabled, "spur", false, "enrich with Spur's threat intelligence data") + flags.BoolVar(&a.Enabled, "spur", false, "annotate with Spur's threat intelligence data") flags.IntVar(&a.Threads, "spur-threads", 100, "how many threads to use for Spur lookups") flags.IntVar(&a.timeoutSecs, "spur-timeout", 2, "timeout for each Spur query, in seconds") } From 4ccbdc7fdaf9224151fe2ec85360902a2b94b64a Mon Sep 17 00:00:00 2001 From: phillip-stephens Date: Tue, 5 May 2026 12:11:31 +1200 Subject: [PATCH 6/6] lint --- cmd/zannotate/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/zannotate/main.go b/cmd/zannotate/main.go index 8a92e7b..dcde95a 100644 --- a/cmd/zannotate/main.go +++ b/cmd/zannotate/main.go @@ -103,7 +103,7 @@ func prepareUsageString(flags *flag.FlagSet) { flags.VisitAll(func(f *flag.Flag) { seen[f.Name] = true }) return seen } - groups := make([]flagGroup, 0, 1 + len(zannotate.Annotators)) // Global options + each annotator + groups := make([]flagGroup, 0, 1+len(zannotate.Annotators)) // Global options + each annotator // Collect the global flags registered above. globalFlags := snapshot() globalNames := slices.Collect(maps.Keys(globalFlags))