Adds --publisher option to four tools (monitors, init, chunks, scrape…

…) but does not implement it

src/apps/chifra/cmd/chunks.go

@@ -70,6 +70,7 @@ func init() {
 	chunksCmd.Flags().BoolVarP(&chunksPkg.GetOptions().Check, "check", "c", false, "check the manifest, index, or blooms for internal consistency")
 	chunksCmd.Flags().BoolVarP(&chunksPkg.GetOptions().Pin, "pin", "i", false, "pin the manifest or each index chunk and bloom")
 	chunksCmd.Flags().BoolVarP(&chunksPkg.GetOptions().Publish, "publish", "p", false, "publish the manifest to the Unchained Index smart contract")
+	chunksCmd.Flags().StringVarP(&chunksPkg.GetOptions().Publisher, "publisher", "P", "trueblocks.eth", "for some query options, the publisher of the index (hidden)")
 	chunksCmd.Flags().Uint64VarP(&chunksPkg.GetOptions().Truncate, "truncate", "n", 0, "truncate the entire index at this block (requires a block identifier) (hidden)")
 	chunksCmd.Flags().BoolVarP(&chunksPkg.GetOptions().Remote, "remote", "r", false, "prior to processing, retreive the manifest from the Unchained Index smart contract")
 	chunksCmd.Flags().StringSliceVarP(&chunksPkg.GetOptions().Belongs, "belongs", "b", nil, "in index mode only, checks the address(es) for inclusion in the given index chunk")
@@ -80,6 +81,7 @@ func init() {
 	chunksCmd.Flags().BoolVarP(&chunksPkg.GetOptions().Deep, "deep", "d", false, "if true, dig more deeply during checking (manifest only)")
 	chunksCmd.Flags().Float64VarP(&chunksPkg.GetOptions().Sleep, "sleep", "s", 0.0, "for --remote pinning only, seconds to sleep between API calls")
 	if os.Getenv("TEST_MODE") != "true" {
+		chunksCmd.Flags().MarkHidden("publisher")
 		chunksCmd.Flags().MarkHidden("truncate")
 		chunksCmd.Flags().MarkHidden("diff")
 	}

src/apps/chifra/cmd/init.go

@@ -63,8 +63,12 @@ func init() {
 
 	initCmd.Flags().BoolVarP(&initPkg.GetOptions().All, "all", "a", false, "in addition to Bloom filters, download full index chunks (recommended)")
 	initCmd.Flags().BoolVarP(&initPkg.GetOptions().DryRun, "dry_run", "d", false, "display the results of the download without actually downloading")
+	initCmd.Flags().StringVarP(&initPkg.GetOptions().Publisher, "publisher", "P", "trueblocks.eth", "the publisher of the index to download (hidden)")
 	initCmd.Flags().Uint64VarP(&initPkg.GetOptions().FirstBlock, "first_block", "F", 0, "do not download any chunks earlier than this block")
 	initCmd.Flags().Float64VarP(&initPkg.GetOptions().Sleep, "sleep", "s", 0.0, "seconds to sleep between downloads")
+	if os.Getenv("TEST_MODE") != "true" {
+		initCmd.Flags().MarkHidden("publisher")
+	}
 	globals.InitGlobals(initCmd, &initPkg.GetOptions().Globals, capabilities)
 
 	initCmd.SetUsageTemplate(UsageWithNotes(notesInit))

src/apps/chifra/cmd/list.go

@@ -66,8 +66,12 @@ func init() {
 	listCmd.Flags().Uint64VarP(&listPkg.GetOptions().FirstRecord, "first_record", "c", 0, "the first record to process")
 	listCmd.Flags().Uint64VarP(&listPkg.GetOptions().MaxRecords, "max_records", "e", 250, "the maximum number of records to process")
 	listCmd.Flags().BoolVarP(&listPkg.GetOptions().Reversed, "reversed", "E", false, "produce results in reverse chronological order")
+	listCmd.Flags().StringVarP(&listPkg.GetOptions().Publisher, "publisher", "P", "trueblocks.eth", "for some query options, the publisher of the index (hidden)")
 	listCmd.Flags().Uint64VarP(&listPkg.GetOptions().FirstBlock, "first_block", "F", 0, "first block to export (inclusive, ignored when freshening)")
 	listCmd.Flags().Uint64VarP(&listPkg.GetOptions().LastBlock, "last_block", "L", 0, "last block to export (inclusive, ignored when freshening)")
+	if os.Getenv("TEST_MODE") != "true" {
+		listCmd.Flags().MarkHidden("publisher")
+	}
 	globals.InitGlobals(listCmd, &listPkg.GetOptions().Globals, capabilities)
 
 	listCmd.SetUsageTemplate(UsageWithNotes(notesList))

src/apps/chifra/cmd/scrape.go

@@ -63,6 +63,7 @@ func init() {
 	scrapeCmd.Flags().Float64VarP(&scrapePkg.GetOptions().Sleep, "sleep", "s", 14, "seconds to sleep between scraper passes")
 	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().StartBlock, "start_block", "l", 0, "first block to visit when scraping (snapped back to most recent snap_to_grid mark)")
 	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().RunCount, "run_count", "u", 0, "run the scraper this many times, then quit (hidden)")
+	scrapeCmd.Flags().StringVarP(&scrapePkg.GetOptions().Publisher, "publisher", "P", "trueblocks.eth", "for some query options, the publisher of the index (hidden)")
 	scrapeCmd.Flags().BoolVarP(&scrapePkg.GetOptions().DryRun, "dry_run", "d", false, "show the configuration that would be applied if run,no changes are made (hidden)")
 	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().Settings.Apps_per_chunk, "apps_per_chunk", "", 200000, "the number of appearances to build into a chunk before consolidating it (hidden)")
 	scrapeCmd.Flags().Uint64VarP(&scrapePkg.GetOptions().Settings.Snap_to_grid, "snap_to_grid", "", 100000, "an override to apps_per_chunk to snap-to-grid at every modulo of this value, this allows easier corrections to the index (hidden)")
@@ -72,6 +73,7 @@ func init() {
 	scrapeCmd.Flags().BoolVarP(&scrapePkg.GetOptions().Settings.Allow_missing, "allow_missing", "", false, "do not report errors for blockchains that contain blocks with zero addresses (hidden)")
 	if os.Getenv("TEST_MODE") != "true" {
 		scrapeCmd.Flags().MarkHidden("run_count")
+		scrapeCmd.Flags().MarkHidden("publisher")
 		scrapeCmd.Flags().MarkHidden("dry_run")
 		scrapeCmd.Flags().MarkHidden("apps_per_chunk")
 		scrapeCmd.Flags().MarkHidden("snap_to_grid")

src/apps/chifra/internal/chunks/options.go

@@ -30,6 +30,7 @@ type ChunksOptions struct {
 	Check      bool                     `json:"check,omitempty"`      // Check the manifest, index, or blooms for internal consistency
 	Pin        bool                     `json:"pin,omitempty"`        // Pin the manifest or each index chunk and bloom
 	Publish    bool                     `json:"publish,omitempty"`    // Publish the manifest to the Unchained Index smart contract
+	Publisher  string                   `json:"publisher,omitempty"`  // For some query options, the publisher of the index
 	Truncate   uint64                   `json:"truncate,omitempty"`   // Truncate the entire index at this block (requires a block identifier)
 	Remote     bool                     `json:"remote,omitempty"`     // Prior to processing, retreive the manifest from the Unchained Index smart contract
 	Belongs    []string                 `json:"belongs,omitempty"`    // In index mode only, checks the address(es) for inclusion in the given index chunk
@@ -47,6 +48,7 @@ type ChunksOptions struct {
 }
 
 var defaultChunksOptions = ChunksOptions{
+	Publisher: "trueblocks.eth",
 	Truncate:  utils.NOPOS,
 	LastBlock: utils.NOPOS,
 	MaxAddrs:  utils.NOPOS,
@@ -59,6 +61,7 @@ func (opts *ChunksOptions) testLog() {
 	logger.TestLog(opts.Check, "Check: ", opts.Check)
 	logger.TestLog(opts.Pin, "Pin: ", opts.Pin)
 	logger.TestLog(opts.Publish, "Publish: ", opts.Publish)
+	logger.TestLog(len(opts.Publisher) > 0, "Publisher: ", opts.Publisher)
 	logger.TestLog(opts.Truncate != utils.NOPOS, "Truncate: ", opts.Truncate)
 	logger.TestLog(opts.Remote, "Remote: ", opts.Remote)
 	logger.TestLog(len(opts.Belongs) > 0, "Belongs: ", opts.Belongs)
@@ -102,6 +105,8 @@ func chunksFinishParseApi(w http.ResponseWriter, r *http.Request) *ChunksOptions
 			opts.Pin = true
 		case "publish":
 			opts.Publish = true
+		case "publisher":
+			opts.Publisher = value[0]
 		case "truncate":
 			opts.Truncate = globals.ToUint64(value[0])
 		case "remote":
@@ -130,6 +135,7 @@ func chunksFinishParseApi(w http.ResponseWriter, r *http.Request) *ChunksOptions
 		}
 	}
 	opts.Conn = opts.Globals.FinishParseApi(w, r, opts.getCaches())
+	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
 	// EXISTING_CODE
@@ -156,6 +162,7 @@ func chunksFinishParse(args []string) *ChunksOptions {
 	defFmt := "txt"
 	opts := GetOptions()
 	opts.Conn = opts.Globals.FinishParse(args, opts.getCaches())
+	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
 	if len(args) > 0 {

src/apps/chifra/internal/init/options.go

@@ -22,6 +22,7 @@ import (
 type InitOptions struct {
 	All        bool                  `json:"all,omitempty"`        // In addition to Bloom filters, download full index chunks (recommended)
 	DryRun     bool                  `json:"dryRun,omitempty"`     // Display the results of the download without actually downloading
+	Publisher  string                `json:"publisher,omitempty"`  // The publisher of the index to download
 	FirstBlock uint64                `json:"firstBlock,omitempty"` // Do not download any chunks earlier than this block
 	Sleep      float64               `json:"sleep,omitempty"`      // Seconds to sleep between downloads
 	Globals    globals.GlobalOptions `json:"globals,omitempty"`    // The global options
@@ -31,12 +32,15 @@ type InitOptions struct {
 	// EXISTING_CODE
 }
 
-var defaultInitOptions = InitOptions{}
+var defaultInitOptions = InitOptions{
+	Publisher: "trueblocks.eth",
+}
 
 // testLog is used only during testing to export the options for this test case.
 func (opts *InitOptions) testLog() {
 	logger.TestLog(opts.All, "All: ", opts.All)
 	logger.TestLog(opts.DryRun, "DryRun: ", opts.DryRun)
+	logger.TestLog(len(opts.Publisher) > 0, "Publisher: ", opts.Publisher)
 	logger.TestLog(opts.FirstBlock != 0, "FirstBlock: ", opts.FirstBlock)
 	logger.TestLog(opts.Sleep != float64(0.0), "Sleep: ", opts.Sleep)
 	opts.Conn.TestLog(opts.getCaches())
@@ -61,6 +65,8 @@ func initFinishParseApi(w http.ResponseWriter, r *http.Request) *InitOptions {
 			opts.All = true
 		case "dryRun":
 			opts.DryRun = true
+		case "publisher":
+			opts.Publisher = value[0]
 		case "firstBlock":
 			opts.FirstBlock = globals.ToUint64(value[0])
 		case "sleep":
@@ -72,6 +78,7 @@ func initFinishParseApi(w http.ResponseWriter, r *http.Request) *InitOptions {
 		}
 	}
 	opts.Conn = opts.Globals.FinishParseApi(w, r, opts.getCaches())
+	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
 	// EXISTING_CODE
@@ -97,6 +104,7 @@ func initFinishParse(args []string) *InitOptions {
 	defFmt := "txt"
 	opts := GetOptions()
 	opts.Conn = opts.Globals.FinishParse(args, opts.getCaches())
+	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
 	if len(args) > 0 {

src/apps/chifra/internal/list/options.go

@@ -32,6 +32,7 @@ type ListOptions struct {
 	FirstRecord uint64                `json:"firstRecord,omitempty"` // The first record to process
 	MaxRecords  uint64                `json:"maxRecords,omitempty"`  // The maximum number of records to process
 	Reversed    bool                  `json:"reversed,omitempty"`    // Produce results in reverse chronological order
+	Publisher   string                `json:"publisher,omitempty"`   // For some query options, the publisher of the index
 	FirstBlock  uint64                `json:"firstBlock,omitempty"`  // First block to export (inclusive, ignored when freshening)
 	LastBlock   uint64                `json:"lastBlock,omitempty"`   // Last block to export (inclusive, ignored when freshening)
 	Globals     globals.GlobalOptions `json:"globals,omitempty"`     // The global options
@@ -43,6 +44,7 @@ type ListOptions struct {
 
 var defaultListOptions = ListOptions{
 	MaxRecords: 250,
+	Publisher:  "trueblocks.eth",
 	LastBlock:  utils.NOPOS,
 }
 
@@ -57,6 +59,7 @@ func (opts *ListOptions) testLog() {
 	logger.TestLog(opts.FirstRecord != 0, "FirstRecord: ", opts.FirstRecord)
 	logger.TestLog(opts.MaxRecords != 250, "MaxRecords: ", opts.MaxRecords)
 	logger.TestLog(opts.Reversed, "Reversed: ", opts.Reversed)
+	logger.TestLog(len(opts.Publisher) > 0, "Publisher: ", opts.Publisher)
 	logger.TestLog(opts.FirstBlock != 0, "FirstBlock: ", opts.FirstBlock)
 	logger.TestLog(opts.LastBlock != 0 && opts.LastBlock != utils.NOPOS, "LastBlock: ", opts.LastBlock)
 	opts.Conn.TestLog(opts.getCaches())
@@ -100,6 +103,8 @@ func listFinishParseApi(w http.ResponseWriter, r *http.Request) *ListOptions {
 			opts.MaxRecords = globals.ToUint64(value[0])
 		case "reversed":
 			opts.Reversed = true
+		case "publisher":
+			opts.Publisher = value[0]
 		case "firstBlock":
 			opts.FirstBlock = globals.ToUint64(value[0])
 		case "lastBlock":
@@ -111,6 +116,7 @@ func listFinishParseApi(w http.ResponseWriter, r *http.Request) *ListOptions {
 		}
 	}
 	opts.Conn = opts.Globals.FinishParseApi(w, r, opts.getCaches())
+	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
 	// EXISTING_CODE
@@ -137,6 +143,7 @@ func listFinishParse(args []string) *ListOptions {
 	defFmt := "txt"
 	opts := GetOptions()
 	opts.Conn = opts.Globals.FinishParse(args, opts.getCaches())
+	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
 	for _, arg := range args {

src/apps/chifra/internal/scrape/options.go

@@ -27,6 +27,7 @@ type ScrapeOptions struct {
 	Sleep      float64                  `json:"sleep,omitempty"`      // Seconds to sleep between scraper passes
 	StartBlock uint64                   `json:"startBlock,omitempty"` // First block to visit when scraping (snapped back to most recent snap_to_grid mark)
 	RunCount   uint64                   `json:"runCount,omitempty"`   // Run the scraper this many times, then quit
+	Publisher  string                   `json:"publisher,omitempty"`  // For some query options, the publisher of the index
 	DryRun     bool                     `json:"dryRun,omitempty"`     // Show the configuration that would be applied if run,no changes are made
 	Settings   scrapeCfg.ScrapeSettings `json:"settings,omitempty"`   // Configuration items for the scrape
 	Globals    globals.GlobalOptions    `json:"globals,omitempty"`    // The global options
@@ -37,7 +38,8 @@ type ScrapeOptions struct {
 }
 
 var defaultScrapeOptions = ScrapeOptions{
-	BlockCnt: 2000,
+	BlockCnt:  2000,
+	Publisher: "trueblocks.eth",
 }
 
 // testLog is used only during testing to export the options for this test case.
@@ -48,6 +50,7 @@ func (opts *ScrapeOptions) testLog() {
 	logger.TestLog(opts.Sleep != float64(14), "Sleep: ", opts.Sleep)
 	logger.TestLog(opts.StartBlock != 0, "StartBlock: ", opts.StartBlock)
 	logger.TestLog(opts.RunCount != 0, "RunCount: ", opts.RunCount)
+	logger.TestLog(len(opts.Publisher) > 0, "Publisher: ", opts.Publisher)
 	logger.TestLog(opts.DryRun, "DryRun: ", opts.DryRun)
 	opts.Settings.TestLog(opts.Globals.Chain, opts.Globals.TestMode)
 	opts.Conn.TestLog(opts.getCaches())
@@ -87,6 +90,8 @@ func scrapeFinishParseApi(w http.ResponseWriter, r *http.Request) *ScrapeOptions
 			opts.StartBlock = globals.ToUint64(value[0])
 		case "runCount":
 			opts.RunCount = globals.ToUint64(value[0])
+		case "publisher":
+			opts.Publisher = value[0]
 		case "dryRun":
 			opts.DryRun = true
 		case "appsPerChunk":
@@ -108,6 +113,7 @@ func scrapeFinishParseApi(w http.ResponseWriter, r *http.Request) *ScrapeOptions
 		}
 	}
 	opts.Conn = opts.Globals.FinishParseApi(w, r, opts.getCaches())
+	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
 	// EXISTING_CODE
@@ -133,6 +139,7 @@ func scrapeFinishParse(args []string) *ScrapeOptions {
 	defFmt := "txt"
 	opts := GetOptions()
 	opts.Conn = opts.Globals.FinishParse(args, opts.getCaches())
+	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
 	if len(args) == 1 && (args[0] == "run" || args[0] == "indexer") {

src/cmd-line-options.csv

@@ -68,6 +68,7 @@ num,group,tags,api_route,tool,longName,hotKey,def_val,is_required,is_customizabl
 12035,apps,Accounts,list,acctExport,first_record,c,0,false,false,true,true,gocmd,flag,<uint64>,the first record to process
 12040,apps,Accounts,list,acctExport,max_records,e,250,false,false,true,true,gocmd,flag,<uint64>,the maximum number of records to process
 12043,apps,Accounts,list,acctExport,reversed,E,,false,false,true,true,gocmd,switch,<boolean>,produce results in reverse chronological order
+12044,apps,Accounts,list,acctExport,publisher,P,trueblocks.eth,false,false,false,false,gocmd,flag,<address>,for some query options&#44; the publisher of the index
 12045,apps,Accounts,list,acctExport,first_block,F,0,false,false,true,true,gocmd,flag,<blknum>,first block to export (inclusive&#44; ignored when freshening)
 12050,apps,Accounts,list,acctExport,last_block,L,NOPOS,false,false,true,true,gocmd,flag,<blknum>,last block to export (inclusive&#44; ignored when freshening)
 12055,apps,Accounts,list,acctExport,,,,false,false,true,true,--,description,,List every appearance of an address anywhere on the chain.
@@ -224,6 +225,7 @@ num,group,tags,api_route,tool,longName,hotKey,def_val,is_required,is_customizabl
 12115,apps,Admin,scrape,blockScrape,sleep,s,14,false,false,true,true,gocmd,flag,<double>,seconds to sleep between scraper passes
 12185,apps,Admin,scrape,blockScrape,start_block,l,0,false,false,true,true,gocmd,flag,<uint64>,first block to visit when scraping (snapped back to most recent snap_to_grid mark)
 12187,apps,Admin,scrape,blockScrape,run_count,u,0,false,false,false,false,gocmd,flag,<uint64>,run the scraper this many times&#44; then quit
+12188,apps,Admin,scrape,blockScrape,publisher,P,trueblocks.eth,false,false,false,false,gocmd,flag,<address>,for some query options&#44; the publisher of the index
 12189,apps,Admin,scrape,blockScrape,dry_run,d,,false,false,false,false,gocmd,switch,<boolean>,show the configuration that would be applied if run&#44;no changes are made
 12160,apps,Admin,scrape,blockScrape,apps_per_chunk,,200000,false,false,false,false,config,flag,<uint64>,the number of appearances to build into a chunk before consolidating it
 12170,apps,Admin,scrape,blockScrape,snap_to_grid,,100000,false,false,false,false,config,flag,<uint64>,an override to apps_per_chunk to snap-to-grid at every modulo of this value&#44; this allows easier corrections to the index
@@ -260,6 +262,7 @@ num,group,tags,api_route,tool,longName,hotKey,def_val,is_required,is_customizabl
 31920,apps,Admin,chunks,chunkMan,check,c,,false,false,true,true,gocmd,switch,<boolean>,check the manifest&#44; index&#44; or blooms for internal consistency
 31935,apps,Admin,chunks,chunkMan,pin,i,,false,false,true,true,gocmd,switch,<boolean>,pin the manifest or each index chunk and bloom
 31940,apps,Admin,chunks,chunkMan,publish,p,,false,false,true,true,gocmd,switch,<boolean>,publish the manifest to the Unchained Index smart contract
+31942,apps,Admin,chunks,chunkMan,publisher,P,trueblocks.eth,false,false,false,false,gocmd,flag,<address>,for some query options&#44; the publisher of the index
 31925,apps,Admin,chunks,chunkMan,truncate,n,,false,false,false,false,gocmd,flag,<blknum>,truncate the entire index at this block (requires a block identifier)
 31945,apps,Admin,chunks,chunkMan,remote,r,,false,false,true,true,gocmd,switch,<boolean>,prior to processing&#44; retreive the manifest from the Unchained Index smart contract
 31950,apps,Admin,chunks,chunkMan,belongs,b,,false,false,true,true,gocmd,flag,list<addr>,in index mode only&#44; checks the address(es) for inclusion in the given index chunk
@@ -282,6 +285,7 @@ num,group,tags,api_route,tool,longName,hotKey,def_val,is_required,is_customizabl
 
 11905,apps,Admin,init,init,all,a,,false,false,true,true,gocmd,switch,<boolean>,in addition to Bloom filters&#44; download full index chunks (recommended)
 11907,apps,Admin,init,init,dry_run,d,,false,false,true,true,gocmd,switch,<boolean>,display the results of the download without actually downloading
+11910,apps,Admin,init,init,publisher,P,trueblocks.eth,false,false,false,false,gocmd,flag,<address>,the publisher of the index to download
 11915,apps,Admin,init,init,first_block,F,0,false,false,true,true,gocmd,flag,<blknum>,do not download any chunks earlier than this block
 11920,apps,Admin,init,init,sleep,s,0.0,false,false,true,true,gocmd,flag,<double>,seconds to sleep between downloads
 11925,apps,Admin,init,init,,,,false,false,true,true,--,description,,Initialize the TrueBlocks system by downloading the Unchained Index from IPFS.