Implements --publisher throughout.

src/apps/chifra/internal/chunks/handle_check.go

@@ -70,12 +70,12 @@ func (opts *ChunksOptions) HandleCheck(blockNums []uint64) error {
 		return fileNames[i] < fileNames[j]
 	})
 
-	cacheManifest, err := manifest.ReadManifest(chain, manifest.FromCache)
+	cacheManifest, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.FromCache)
 	if err != nil {
 		return err
 	}
 
-	remoteManifest, err := manifest.ReadManifest(chain, manifest.FromContract)
+	remoteManifest, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.FromContract)
 	if err != nil {
 		return err
 	}

src/apps/chifra/internal/chunks/handle_manifest.go

@@ -20,7 +20,7 @@ var sourceMap = map[bool]manifest.Source{
 func (opts *ChunksOptions) HandleManifest(blockNums []uint64) error {
 	chain := opts.Globals.Chain
 	testMode := opts.Globals.TestMode
-	man, err := manifest.ReadManifest(chain, sourceMap[opts.Remote])
+	man, err := manifest.ReadManifest(chain, opts.PublisherAddr, sourceMap[opts.Remote])
 	if err != nil {
 		return err
 	}

src/apps/chifra/internal/chunks/handle_truncate.go

@@ -64,7 +64,7 @@ func (opts *ChunksOptions) HandleTruncate(blockNums []uint64) error {
 
 			testRange := base.FileRange{First: opts.Truncate, Last: utils.NOPOS}
 			if rng.Intersects(testRange) {
-				if err = manifest.RemoveChunk(chain, index.ToBloomPath(path), index.ToIndexPath(path)); err != nil {
+				if err = manifest.RemoveChunk(chain, opts.PublisherAddr, index.ToBloomPath(path), index.ToIndexPath(path)); err != nil {
 					return false, err
 				}
 				nChunksRemoved++

src/apps/chifra/internal/chunks/options.go

@@ -44,6 +44,7 @@ type ChunksOptions struct {
 	Conn       *rpc.Connection          `json:"conn,omitempty"`       // The connection to the RPC server
 	BadFlag    error                    `json:"badFlag,omitempty"`    // An error flag if needed
 	// EXISTING_CODE
+	PublisherAddr base.Address
 	// EXISTING_CODE
 }
 
@@ -138,6 +139,7 @@ func chunksFinishParseApi(w http.ResponseWriter, r *http.Request) *ChunksOptions
 	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
+	opts.PublisherAddr = base.HexToAddress(opts.Publisher)
 	// EXISTING_CODE
 	opts.Belongs, _ = opts.Conn.GetEnsAddresses(opts.Belongs)
 
@@ -165,6 +167,7 @@ func chunksFinishParse(args []string) *ChunksOptions {
 	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
+	opts.PublisherAddr = base.HexToAddress(opts.Publisher)
 	if len(args) > 0 {
 		opts.Mode = args[0]
 		for i, arg := range args {

src/apps/chifra/internal/chunks/validate.go

@@ -133,6 +133,13 @@ func (opts *ChunksOptions) validateChunks() error {
 		return err
 	}
 
+	if len(opts.Publisher) > 0 {
+		err := validate.ValidateExactlyOneAddr([]string{opts.Publisher})
+		if err != nil {
+			return err
+		}
+	}
+
 	err = validate.ValidateIdentifiers(
 		chain,
 		opts.Blocks,

src/apps/chifra/internal/export/handle_neighbors.go

@@ -141,7 +141,7 @@ bool isSame(const CReverseAppMapEntry* a, const CReverseAppMapEntry* b) {
 
 //-----------------------------------------------------------------------
 bool assignReason(const CName& accountedFor, CAppearance& app, const CTransaction& trans) {
-    if (app.transactionIndex > 99996) {  // leave this here for searching: 99999
+    if (app.transactionIndex > types.ExternalReward) {  // leave this here for searching: types.BlockReward
         app.reason = "miner";
         return true;
     }

src/apps/chifra/internal/globals/options.go

@@ -15,6 +15,7 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/rpc"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/tslib"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/unchained"
 	"github.com/spf13/cobra"
 )
 
@@ -196,8 +197,11 @@ func (opts *GlobalOptions) FinishParseApi(w http.ResponseWriter, r *http.Request
 	}
 
 	if config.IsChainConfigured(opts.Chain) {
-		if err := tslib.EstablishTsFile(opts.Chain); err != nil {
-			logger.Error("Could not establish ts file:", err)
+		// TODO: #3219 Either this needs to be an option or PreferredPublisher needs to be configurable
+		// TODO: Why do we need to do this here?
+		publisher := unchained.GetPreferredPublisher()
+		if err := tslib.EstablishTsFile(opts.Chain, publisher); err != nil {
+			logger.Warn(err)
 		}
 		return rpc.NewConnection(opts.Chain, opts.Cache && !opts.ShowRaw, caches)
 	} else {
@@ -222,8 +226,11 @@ func (opts *GlobalOptions) FinishParse(args []string, caches map[string]bool) *r
 	}
 
 	if config.IsChainConfigured(opts.Chain) {
-		if err := tslib.EstablishTsFile(opts.Chain); err != nil {
-			logger.Error("Could not establish ts file:", err)
+		// TODO: #3219 Either this needs to be an option or PreferredPublisher needs to be configurable
+		// TODO: Why do we need to do this here?
+		publisher := unchained.GetPreferredPublisher()
+		if err := tslib.EstablishTsFile(opts.Chain, publisher); err != nil {
+			logger.Warn(err)
 		}
 		return rpc.NewConnection(opts.Chain, opts.Cache && !opts.ShowRaw, caches)
 	} else {

src/apps/chifra/internal/init/handle_dryrun.go

@@ -16,7 +16,7 @@ import (
 func (opts *InitOptions) HandleDryRun() error {
 	chain := opts.Globals.Chain
 
-	remoteManifest, err := manifest.ReadManifest(chain, manifest.FromContract)
+	remoteManifest, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.FromContract)
 	if err != nil {
 		return err
 	}

src/apps/chifra/internal/init/handle_init.go

@@ -32,7 +32,7 @@ func (opts *InitOptions) HandleInit() error {
 	// scraper starts, it starts on the correct block.
 	_ = index.CleanTempIndexFolders(chain, []string{"ripe", "unripe", "staging"})
 
-	remoteManifest, err := manifest.ReadManifest(chain, manifest.FromContract)
+	remoteManifest, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.FromContract)
 	if err != nil {
 		return err
 	}

src/apps/chifra/internal/init/options.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/internal/globals"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/caps"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/rpc"
@@ -29,6 +30,7 @@ type InitOptions struct {
 	Conn       *rpc.Connection       `json:"conn,omitempty"`       // The connection to the RPC server
 	BadFlag    error                 `json:"badFlag,omitempty"`    // An error flag if needed
 	// EXISTING_CODE
+	PublisherAddr base.Address
 	// EXISTING_CODE
 }
 
@@ -81,6 +83,7 @@ func initFinishParseApi(w http.ResponseWriter, r *http.Request) *InitOptions {
 	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
+	opts.PublisherAddr = base.HexToAddress(opts.Publisher)
 	// EXISTING_CODE
 
 	return opts
@@ -107,6 +110,7 @@ func initFinishParse(args []string) *InitOptions {
 	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
+	opts.PublisherAddr = base.HexToAddress(opts.Publisher)
 	if len(args) > 0 {
 		opts.BadFlag = validate.Usage("Invalid argument ({0}).", args[0])
 	}

src/apps/chifra/internal/init/validate.go

@@ -26,5 +26,12 @@ func (opts *InitOptions) validateInit() error {
 		return validate.Usage("integration testing was skipped for chifra init")
 	}
 
+	if len(opts.Publisher) > 0 {
+		err := validate.ValidateExactlyOneAddr([]string{opts.Publisher})
+		if err != nil {
+			return err
+		}
+	}
+
 	return opts.Globals.Validate()
 }

src/apps/chifra/internal/list/handle_freshen.go

@@ -271,7 +271,7 @@ func (updater *MonitorUpdate) visitChunkToFreshenFinal(fileName string, resultCh
 
 	indexFilename := index.ToIndexPath(fileName)
 	if !file.FileExists(indexFilename) {
-		_, err := index.EstablishIndexChunk(updater.Options.Globals.Chain, bl.Range)
+		_, err := index.EstablishIndexChunk(updater.Options.Globals.Chain, updater.Options.PublisherAddr, bl.Range)
 		if err != nil {
 			results = append(results, index.AppearanceResult{Range: bl.Range, Err: err})
 			return

src/apps/chifra/internal/list/list_integration_test.go

@@ -20,7 +20,7 @@ import (
 )
 
 func Test_HandleFreshenMonitors(t *testing.T) {
-	tslib.EstablishTsFile(utils.GetTestChain())
+	tslib.EstablishTsFile(utils.GetTestChain(), base.GetTestPublisher())
 	opts := globals.GlobalOptions{}
 	opts.Chain = "mainnet"
 	listOpts := ListOptions{

src/apps/chifra/internal/list/options.go

@@ -39,6 +39,7 @@ type ListOptions struct {
 	Conn        *rpc.Connection       `json:"conn,omitempty"`        // The connection to the RPC server
 	BadFlag     error                 `json:"badFlag,omitempty"`     // An error flag if needed
 	// EXISTING_CODE
+	PublisherAddr base.Address
 	// EXISTING_CODE
 }
 
@@ -119,6 +120,7 @@ func listFinishParseApi(w http.ResponseWriter, r *http.Request) *ListOptions {
 	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
+	opts.PublisherAddr = base.HexToAddress(opts.Publisher)
 	// EXISTING_CODE
 	opts.Addrs, _ = opts.Conn.GetEnsAddresses(opts.Addrs)
 
@@ -146,6 +148,7 @@ func listFinishParse(args []string) *ListOptions {
 	opts.Publisher, _ = opts.Conn.GetEnsAddress(opts.Publisher)
 
 	// EXISTING_CODE
+	opts.PublisherAddr = base.HexToAddress(opts.Publisher)
 	for _, arg := range args {
 		if base.IsValidAddress(arg) {
 			opts.Addrs = append(opts.Addrs, arg)

src/apps/chifra/internal/list/validate.go

@@ -67,6 +67,13 @@ func (opts *ListOptions) validateList() error {
 		}
 	}
 
+	if len(opts.Publisher) > 0 {
+		err := validate.ValidateExactlyOneAddr([]string{opts.Publisher})
+		if err != nil {
+			return err
+		}
+	}
+
 	// Note that this does not return if the index is not initialized
 	if err := index.IndexIsInitialized(chain); err != nil {
 		if opts.Globals.IsApiMode() {

src/apps/chifra/internal/scrape/scrape_consolidate.go

@@ -123,15 +123,15 @@ func (bm *BlazeManager) Consolidate() (bool, error) {
 				}
 			}
 
-			indexPath := config.PathToIndex(chain) + "finalized/" + curRange.String() + ".bin"
-			if report, err := index.WriteChunk(chain, indexPath, appMap, len(appearances), bm.opts.Pin, bm.opts.Remote); err != nil {
-				return false, err
-			} else if report == nil {
-				logger.Fatal("Should not happen, write chunk returned empty report")
-			} else {
-				report.Snapped = isSnap
-				report.Report()
-			}
+			// indexPath := config.PathToIndex(chain) + "finalized/" + curRange.String() + ".bin"
+			// if report, err := index.WriteChunk(chain, opts.PublisherAddr, indexPath, appMap, len(appearances), bm.opts.Pin, bm.opts.Remote); err != nil {
+			// 	return false, err
+			// } else if report == nil {
+			// 	logger.Fatal("Should not happen, write chunk returned empty report")
+			// } else {
+			// 	report.Snapped = isSnap
+			// 	report.Report()
+			// }
 
 			curRange.First = curRange.Last + 1
 			appearances = []string{}

src/apps/chifra/internal/scrape/scrape_prepare.go

@@ -10,7 +10,6 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/index"
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/prefunds"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/tslib"
 )
@@ -54,16 +53,16 @@ func (opts *ScrapeOptions) Prepare() (ok bool, err error) {
 	})
 	_ = tslib.Append(chain, array)
 
-	logger.Info("Writing block zero allocations for", len(prefunds), "prefunds, nAddresses:", len(appMap))
-	indexPath := index.ToIndexPath(bloomPath)
-	if report, err := index.WriteChunk(chain, indexPath, appMap, len(prefunds), opts.Pin, opts.Remote); err != nil {
-		return false, err
-	} else if report == nil {
-		logger.Fatal("Should not happen, write chunk returned empty report")
-	} else {
-		report.Snapped = true // assumes block zero is a snap to grid (which it is in a sense)
-		report.Report()
-	}
+	// logger.Info("Writing block zero allocations for", len(prefunds), "prefunds, nAddresses:", len(appMap))
+	// indexPath := index.ToIndexPath(bloomPath)
+	// if report, err := index.WriteChunk(chain, indexPath, appMap, len(prefunds), opts.Pin, opts.Remote); err != nil {
+	// 	return false, err
+	// } else if report == nil {
+	// 	logger.Fatal("Should not happen, write chunk returned empty report")
+	// } else {
+	// 	report.Snapped = true // assumes block zero is a snap to grid (which it is in a sense)
+	// 	report.Report()
+	// }
 
 	return true, nil
 }

src/apps/chifra/pkg/index/chunk_write.go

@@ -47,7 +47,7 @@ func (c *WriteChunkReport) Report() {
 	}
 }
 
-func WriteChunk(chain string, fileName string, addrAppearanceMap AddressAppearanceMap, nApps int, pin, remote bool) (*WriteChunkReport, error) {
+func WriteChunk(chain string, publisher base.Address, fileName string, addrAppearanceMap AddressAppearanceMap, nApps int, pin, remote bool) (*WriteChunkReport, error) {
 	// We're going to build two tables. An addressTable and an appearanceTable. We do this as we spin
 	// through the map
 
@@ -168,7 +168,7 @@ func WriteChunk(chain string, fileName string, addrAppearanceMap AddressAppearan
 			report.PinRecord.BloomHash = rec.BloomHash
 			report.PinRecord.IndexSize = rec.IndexSize
 			report.PinRecord.BloomSize = rec.BloomSize
-			return &report, manifest.UpdateManifest(chain, rec)
+			return &report, manifest.UpdateManifest(chain, publisher, rec)
 
 		} else {
 			return nil, err

src/apps/chifra/pkg/index/establish.go

@@ -25,10 +25,10 @@ import (
 
 // EstablishIndexChunk a filename to an index portion, finds the correspoding CID (hash)
 // entry in the manifest, and downloads the index chunk to the local drive
-func EstablishIndexChunk(chain string, fileRange base.FileRange) (bool, error) {
+func EstablishIndexChunk(chain string, publisher base.Address, fileRange base.FileRange) (bool, error) {
 	exists, fileName := fileRange.RangeToFilename(chain)
 
-	chunkManifest, err := manifest.ReadManifest(chain, manifest.FromCache)
+	chunkManifest, err := manifest.ReadManifest(chain, publisher, manifest.FromCache)
 	if err != nil {
 		return exists, err
 	}

src/apps/chifra/pkg/index/uniq_appearances.go

@@ -11,14 +11,14 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
-// // UniqFromWithdrawals extracts addresses from an array of receipts
-// func UniqFromWithdrawals(chain string, withdrawals []types.SimpleWithdrawal, bn base.Blknum, addrMap AddressBooleanMap) (err error) {
-// 	for _, withdrawal := range withdrawals {
-// 		// logger.Info("Adding withdrawl", withdrawal.Address.Hex(), bn, types.Withdrawal)
-// 		addAddressToMaps(withdrawal.Address.Hex(), bn, types.Withdrawal, addrMap)
-// 	}
-// 	return nil
-// }
+// UniqFromWithdrawals extracts addresses from an array of receipts
+func UniqFromWithdrawals(chain string, withdrawals []types.SimpleWithdrawal, bn base.Blknum, addrMap AddressBooleanMap) (err error) {
+	for _, withdrawal := range withdrawals {
+		// logger.Info("Adding withdrawl", withdrawal.Address.Hex(), bn, types.Withdrawal)
+		addAddressToMaps(withdrawal.Address.Hex(), bn, types.Withdrawal, addrMap)
+	}
+	return nil
+}
 
 // UniqFromReceipts extracts addresses from an array of receipts
 func UniqFromReceipts(chain string, receipts []types.SimpleReceipt, addrMap AddressBooleanMap) (err error) {
@@ -83,11 +83,11 @@ func UniqFromTraces(chain string, traces []types.SimpleTrace, addrMap AddressBoo
 					// Early clients allowed misconfigured miner settings with address
 					// 0x0 (reward got burned). We enter a false record with a false tx_id
 					// to account for this.
-					author = "0xdeaddeaddeaddeaddeaddeaddeaddeaddeaddead"
-					addAddressToMaps(author, bn, 99997, addrMap)
+					author = base.SentinalAddr.Hex()
+					addAddressToMaps(author, bn, types.MisconfigReward, addrMap)
 
 				} else {
-					addAddressToMaps(author, bn, 99999, addrMap)
+					addAddressToMaps(author, bn, types.BlockReward, addrMap)
 
 				}
 
@@ -98,18 +98,18 @@ func UniqFromTraces(chain string, traces []types.SimpleTrace, addrMap AddressBoo
 					// Early clients allowed misconfigured miner settings with address
 					// 0x0 (reward got burned). We enter a false record with a false tx_id
 					// to account for this.
-					author = "0xdeaddeaddeaddeaddeaddeaddeaddeaddeaddead"
-					addAddressToMaps(author, bn, 99998, addrMap)
+					author = base.SentinalAddr.Hex()
+					addAddressToMaps(author, bn, types.UncleReward, addrMap)
 
 				} else {
-					addAddressToMaps(author, bn, 99998, addrMap)
+					addAddressToMaps(author, bn, types.UncleReward, addrMap)
 
 				}
 
 			} else if trace.Action.RewardType == "external" {
 				// This only happens in xDai as far as we know...
 				author := trace.Action.Author.Hex()
-				addAddressToMaps(author, bn, 99996, addrMap)
+				addAddressToMaps(author, bn, types.ExternalReward, addrMap)
 
 			} else {
 				fmt.Println("Unknown reward type", trace.Action.RewardType)