Add claim count for each channel

.travis.yml

@@ -9,7 +9,7 @@ language: go
 # in the next version of Go. Don't worry! Later we declare that test runs
 # are allowed to fail on Go tip.
 go:
-  - 1.13.4
+  - 1.15.2
   - master
 
 # Skip the install step. Don't `go get` dependencies. Only build with the

daemon/daemon.go

@@ -69,6 +69,7 @@ func initJobs() {
 	scheduleJob(jobs.ValidateChain, "Validate Chain", 24*time.Hour)
 	scheduleJob(jobs.SyncAddressBalancesJob, "Address Balance Sync", 24*time.Hour)
 	scheduleJob(jobs.TransactionValueASync, "Transaction Value Sync", 24*time.Hour)
+	scheduleJob(jobs.SyncClaimsInChannelJob, "Claim Count in Channel Sync", 24*time.Hour)
 	//ChainSync job should never be run later than 2.5 minutes or its possible it will never loop back due to coinbase time
 	scheduleJob(jobs.ChainSyncAsync, "Chain Sync", 5*time.Second)
 }

daemon/jobs/claimcntsync.go

@@ -0,0 +1 @@
+package jobs

daemon/jobs/valuesync.go

@@ -2,14 +2,16 @@ package jobs
 
 import (
 	"github.com/lbryio/chainquery/model"
-	"github.com/lbryio/lbry.go/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/sirupsen/logrus"
+	"github.com/volatiletech/null"
 	"github.com/volatiletech/sqlboiler/boil"
 	"github.com/volatiletech/sqlboiler/queries/qm"
 )
 
 const syncTransactionValues = "SyncTransactionValues: "
 const syncAddressBalances = "SyncAddressBalances: "
+const syncClaimsInChannel = "SyncClaimsInChannel: "
 
 //SyncAddressBalancesJob runs the SyncAddressBalances as a background job.
 func SyncAddressBalancesJob() {
@@ -21,6 +23,16 @@ func SyncAddressBalancesJob() {
 	}()
 }
 
+// SyncClaimsInChannelJob runs the SyncClaimsInChannel as a background job.
+func SyncClaimsInChannelJob() {
+	go func() {
+		err := SyncClaimCntInChannel()
+		if err != nil {
+			logrus.Error(syncClaimsInChannel, err)
+		}
+	}()
+}
+
 //TransactionValueSync synchronizes the transaction value column due to a bug in mysql related to triggers.
 //https://bugs.mysql.com/bug.php?id=11472
 func TransactionValueSync() {
@@ -108,6 +120,9 @@ func SyncTransactionValue() (int64, error) {
 	}
 	latestHeight := int(latestBlock.Height)
 	updateIncrement := 5000
+	if updateIncrement >= latestHeight {
+		updateIncrement = latestHeight
+	}
 	for i := 0; i < latestHeight/updateIncrement; i++ {
 		from = i * updateIncrement
 		to = (i + 1) * updateIncrement
@@ -132,3 +147,62 @@ func SyncTransactionValue() (int64, error) {
 	return affected, nil
 
 }
+
+// SyncClaimCntInChannel will sync up the number of claims that are part of a particular channel. This can be used as a
+// calculated column in the claim table to get this figure fast in a query.
+func SyncClaimCntInChannel() error {
+
+	t := model.TableNames
+	c := model.ClaimColumns
+
+	query := `SELECT COUNT(*) FROM ` + t.Claim + ` WHERE ` + t.Claim + `.` + c.PublisherID + ` = ?`
+
+	var from int
+	var to int
+
+	latestBlock, err := model.Blocks(qm.Select(model.BlockColumns.Height), qm.OrderBy(model.BlockColumns.Height+" DESC")).OneG()
+	if err != nil {
+		return errors.Prefix(syncClaimsInChannel, err)
+	}
+	if latestBlock.Height == 0 {
+		return errors.Prefix(syncClaimsInChannel, errors.Err("latest height = 0 "))
+	}
+	latestHeight := int(latestBlock.Height)
+	updateIncrement := 5000
+	if updateIncrement >= latestHeight {
+		updateIncrement = latestHeight
+	}
+	for i := 0; i < latestHeight/updateIncrement; i++ {
+		from = i * updateIncrement
+		to = (i + 1) * updateIncrement
+		if to > latestHeight {
+			to = latestHeight
+		}
+		channelsToProcess, err := model.Claims(
+			model.ClaimWhere.Height.GTE(uint(from)),
+			model.ClaimWhere.Height.LTE(uint(to)),
+			model.ClaimWhere.ClaimType.EQ(2)).AllG()
+		if err != nil {
+			return errors.Prefix(syncClaimsInChannel, err)
+		}
+		for _, channel := range channelsToProcess {
+			result := boil.GetDB().QueryRow(query, channel.ClaimID)
+			if err != nil {
+				return errors.Prefix(syncClaimsInChannel, err)
+			}
+			var cnt null.Uint64
+			err := result.Scan(&cnt)
+			if err != nil {
+				return errors.Prefix(syncClaimsInChannel, err)
+			}
+			channel.ClaimCount = int64(cnt.Uint64)
+			err = channel.UpdateG(boil.Whitelist(c.ClaimCount))
+			if err != nil {
+				return errors.Prefix(syncClaimsInChannel, err)
+			}
+		}
+	}
+
+	return nil
+
+}

daemon/processing/block.go

@@ -342,7 +342,7 @@ func handleFailure(err error, manager *txSyncManager) {
 func getBlockToProcess(height *uint64) (*lbrycrd.GetBlockResponse, error) {
 	hash, err := lbrycrd.GetBlockHash(*height)
 	if err != nil {
-		return nil, errors.Prefix("GetBlockHash Error("+string(*height)+"): ", err)
+		return nil, errors.Prefix(fmt.Sprintf("GetBlockHash Error(%d): ", *height), err)
 	}
 	jsonBlock, err := lbrycrd.GetBlock(*hash)
 	if err != nil {

datastore/datastore.go

@@ -3,6 +3,7 @@ package datastore
 import (
 	"github.com/lbryio/chainquery/model"
 	"github.com/lbryio/lbry.go/extras/errors"
+	"github.com/volatiletech/null"
 
 	"time"
 
@@ -352,8 +353,8 @@ func PutTag(tag *model.Tag) error {
 // GetClaimTag makes creating,retrieving,updating the model type simplified.
 func GetClaimTag(tagID uint64, claimID string) *model.ClaimTag {
 	defer util.TimeTrack(time.Now(), "GetClaimTag", "mysqlprofile")
-	tagIDMatch := qm.Where(model.ClaimTagColumns.TagID+"=?", tagID)
-	claimIDMatch := qm.Where(model.ClaimTagColumns.ClaimID+"=?", claimID)
+	claimIDMatch := model.ClaimTagWhere.ClaimID.EQ(claimID)
+	tagIDMatch := model.ClaimTagWhere.TagID.EQ(null.Uint64From(tagID))
 
 	exists, err := model.ClaimTags(tagIDMatch, claimIDMatch).ExistsG()
 	if err != nil {
@@ -376,7 +377,12 @@ func PutClaimTag(claimTag *model.ClaimTag) error {
 	if claimTag != nil {
 
 		var err error
-		exists, err := model.ClaimTagExistsG(claimTag.ID)
+		if !model.Claims(model.ClaimWhere.ClaimID.EQ(claimTag.ClaimID)).ExistsGP() {
+			logrus.Error("Failed to find claim ", claimTag.ClaimID)
+		}
+		claimIDMatch := model.ClaimTagWhere.ClaimID.EQ(claimTag.ClaimID)
+		tagIDMatch := model.ClaimTagWhere.TagID.EQ(claimTag.TagID)
+		exists, err := model.ClaimTags(claimIDMatch, tagIDMatch).ExistsG()
 		if err != nil {
 			return errors.Prefix("Datastore(PUTCLAIMTAG): ", err)
 		}

e2e/e2e.go

@@ -29,8 +29,10 @@ func StartE2ETesting() {
 	increment(1)
 	jobs.ClaimTrieSync()
 	jobs.CertificateSync()
+	exitOnErr(jobs.SyncClaimCntInChannel())
 	time.Sleep(2 * time.Second)
 	exitOnErr(checkCertValid([]string{"claim1", "claim2", "claim3"}))
+	testClaimCount()
 	testImageMetadata()
 	testVideoMetaData()
 	daemon.ShutdownDaemon()
@@ -155,3 +157,14 @@ func checkForBlock(blockHash string) (bool, error) {
 	}
 	return false, nil
 }
+
+func testClaimCount() {
+	channel, err := model.Claims(model.ClaimWhere.Name.EQ("@MyChannel")).OneG()
+	exitOnErr(err)
+	if channel == nil {
+		exit(1, errors.Err("Could not find channel @MyChannel"))
+	}
+	if channel.ClaimCount != 3 {
+		exit(1, errors.Err("@MyChannel only has %d claims and should have 3", channel.ClaimCount))
+	}
+}

e2e/e2e.sh

@@ -1,25 +1,10 @@
 #!/usr/bin/env bash
 
 set -e
-echo 'Running e2e tests...'
-./scripts/build.sh
-mysql -u root -e 'DROP DATABASE IF EXISTS chainquery_e2e_test;'
-mysql -u root -e 'CREATE DATABASE IF NOT EXISTS chainquery_e2e_test;'
-mysql -u root -e "GRANT ALL ON chainquery_e2e_test.* TO 'lbry'@'localhost';"
-cd e2e
-docker-compose stop
-docker-compose rm -f
-if [ -d ../persist ]; then rm -r ../persist; fi
-mkdir ../persist
-docker-compose pull
-docker-compose up -d lbrycrd
-docker ps
-sleep 20
-echo "Generating 800 blocks"
-docker-compose exec lbrycrd lbrycrd-cli --conf=/etc/lbry/lbrycrd.conf generate 800
-echo "Running Chainquery e2e test"
-../bin/chainquery e2e --configpath=$PWD/e2e
+./e2e/prepare_e2e.sh
+./bin/chainquery e2e --configpath=$PWD/e2e
 echo $?
+cd e2e
 docker-compose stop lbrycrd
 if [ -d persist ]; then rm -r persist; fi #Remove this if you want to debug the lbrycrd data, debug docker or see files grabbed.
 echo "Finished e2e test"

e2e/prepare_e2e.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -e
+echo 'Running e2e tests...'
+if [ -d persist ]; then rm -r persist; fi #Remove this if you want to debug the lbrycrd data, debug docker or see files grabbed.
+./scripts/build.sh
+mysql -u root -e 'DROP DATABASE IF EXISTS chainquery_e2e_test;'
+mysql -u root -e 'CREATE DATABASE IF NOT EXISTS chainquery_e2e_test;'
+mysql -u root -e "GRANT ALL ON chainquery_e2e_test.* TO 'lbry'@'localhost';"
+cd e2e
+docker-compose stop
+docker-compose rm -f
+if [ -d ../persist ]; then rm -r ../persist; fi
+mkdir ../persist
+docker-compose pull
+docker-compose up -d lbrycrd
+docker ps
+sleep 20
+echo "Generating 800 blocks"
+docker-compose exec lbrycrd lbrycrd-cli --conf=/etc/lbry/lbrycrd.conf generate 800

go.mod

@@ -1,6 +1,6 @@
 module github.com/lbryio/chainquery
 
-go 1.12
+go 1.15
 
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
 

migration/024_claims_in_channel.sql

@@ -0,0 +1,8 @@
+-- +migrate Up
+-- +migrate StatementBegin
+ALTER TABLE claim ADD COLUMN claim_count BIGINT NOT NULL DEFAULT 0;
+-- +migrate StatementEnd
+
+-- +migrate StatementBegin
+ALTER TABLE claim ADD INDEX (claim_count);
+-- +migrate StatementEnd