finishing porting omni watcher to work with light sync; split into fu…

…ll, light,

and shared directories and refactor as much into shared; finish
lightSync omni watcher tests

README.md

@@ -9,10 +9,10 @@
 Vulcanize DB is a set of tools that make it easier for developers to write application-specific indexes and caches for dapps built on Ethereum.
 
 ## Dependencies
- - Go 1.9+
+ - Go 1.11+
  - Postgres 10
  - Ethereum Node
-   - [Go Ethereum](https://ethereum.github.io/go-ethereum/downloads/) (1.8+)
+   - [Go Ethereum](https://ethereum.github.io/go-ethereum/downloads/) (1.8.18+)
    - [Parity 1.8.11+](https://github.com/paritytech/parity/releases)
 
 ## Project Setup

cmd/lightOmniWatcher.go

@@ -17,11 +17,8 @@
 package cmd
 
 import (
-	"bufio"
 	"fmt"
 	"log"
-	"os"
-	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -59,29 +56,6 @@ func lightOmniWatcher() {
 		log.Fatal("Contract address required")
 	}
 
-	if len(contractEvents) == 0 || len(contractMethods) == 0 {
-		var str string
-		for str != "y" {
-			reader := bufio.NewReader(os.Stdin)
-			if len(contractEvents) == 0 && len(contractMethods) == 0 {
-				fmt.Print("Warning: no events or methods specified.\n Proceed to watch every event and poll no methods? (Y/n)\n> ")
-			} else if len(contractEvents) == 0 {
-				fmt.Print("Warning: no events specified.\n Proceed to watch every event? (Y/n)\n> ")
-			} else {
-				fmt.Print("Warning: no methods specified.\n Proceed to poll no methods? (Y/n)\n> ")
-			}
-			resp, err := reader.ReadBytes('\n')
-			if err != nil {
-				log.Fatal(err)
-			}
-
-			str = strings.ToLower(string(resp))
-			if str == "n" {
-				return
-			}
-		}
-	}
-
 	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
 
@@ -114,13 +88,13 @@ func lightOmniWatcher() {
 func init() {
 	rootCmd.AddCommand(lightOmniWatcherCmd)
 
-	omniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for")
-	omniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address")
-	omniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched")
-	omniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "contract-methods", "m", nil, "Subset of methods to poll; by default no methods are polled")
-	omniWatcherCmd.Flags().StringArrayVarP(&eventAddrs, "event-filter-addresses", "f", []string{}, "Account addresses to persist event data for; default is to persist for all found token holder addresses")
-	omniWatcherCmd.Flags().StringArrayVarP(&methodAddrs, "method-filter-addresses", "g", []string{}, "Account addresses to poll methods with; default is to poll with all found token holder addresses")
-	omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)
-	omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists")
-	omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block")
+	lightOmniWatcherCmd.Flags().StringVarP(&contractAddress, "contract-address", "a", "", "Single address to generate watchers for")
+	lightOmniWatcherCmd.Flags().StringArrayVarP(&contractAddresses, "contract-addresses", "l", []string{}, "list of addresses to use; warning: watcher targets the same events and methods for each address")
+	lightOmniWatcherCmd.Flags().StringArrayVarP(&contractEvents, "contract-events", "e", []string{}, "Subset of events to watch; by default all events are watched")
+	lightOmniWatcherCmd.Flags().StringArrayVarP(&contractMethods, "contract-methods", "m", nil, "Subset of methods to poll; by default no methods are polled")
+	lightOmniWatcherCmd.Flags().StringArrayVarP(&eventAddrs, "event-filter-addresses", "f", []string{}, "Account addresses to persist event data for; default is to persist for all found token holder addresses")
+	lightOmniWatcherCmd.Flags().StringArrayVarP(&methodAddrs, "method-filter-addresses", "g", []string{}, "Account addresses to poll methods with; default is to poll with all found token holder addresses")
+	lightOmniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)
+	lightOmniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists")
+	lightOmniWatcherCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block")
 }

cmd/omniWatcher.go

@@ -17,11 +17,8 @@
 package cmd
 
 import (
-	"bufio"
 	"fmt"
 	"log"
-	"os"
-	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -59,29 +56,6 @@ func omniWatcher() {
 		log.Fatal("Contract address required")
 	}
 
-	if len(contractEvents) == 0 || len(contractMethods) == 0 {
-		var str string
-		for str != "y" {
-			reader := bufio.NewReader(os.Stdin)
-			if len(contractEvents) == 0 && len(contractMethods) == 0 {
-				fmt.Print("Warning: no events or methods specified.\n Proceed to watch every event and poll no methods? (Y/n)\n> ")
-			} else if len(contractEvents) == 0 {
-				fmt.Print("Warning: no events specified.\n Proceed to watch every event? (Y/n)\n> ")
-			} else {
-				fmt.Print("Warning: no methods specified.\n Proceed to poll no methods? (Y/n)\n> ")
-			}
-			resp, err := reader.ReadBytes('\n')
-			if err != nil {
-				log.Fatal(err)
-			}
-
-			str = strings.ToLower(string(resp))
-			if str == "n" {
-				return
-			}
-		}
-	}
-
 	ticker := time.NewTicker(5 * time.Second)
 	defer ticker.Stop()
 
@@ -122,5 +96,5 @@ func init() {
 	omniWatcherCmd.Flags().StringArrayVarP(&methodAddrs, "method-filter-addresses", "g", []string{}, "Account addresses to poll methods with; default is to poll with all found token holder addresses")
 	omniWatcherCmd.Flags().StringVarP(&network, "network", "n", "", `Network the contract is deployed on; options: "ropsten", "kovan", and "rinkeby"; default is mainnet"`)
 	omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "starting-block-number", "s", 0, "Block to begin watching- default is first block the contract exists")
-	omniWatcherCmd.Flags().Int64VarP(&startingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block")
+	omniWatcherCmd.Flags().Int64VarP(&endingBlockNumber, "ending-block-number", "d", -1, "Block to end watching- default is most recent block")
 }

db/migrations/1532468317_create_checked_headers_table.up.sql

@@ -1,5 +1,4 @@
 CREATE TABLE public.checked_headers (
   id                  SERIAL PRIMARY KEY,
-  header_id           INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE,
-  price_feeds_checked BOOLEAN NOT NULL DEFAULT FALSE
+  header_id           INTEGER UNIQUE NOT NULL REFERENCES headers (id) ON DELETE CASCADE
 );

pkg/omni/full/converter/converter.go

@@ -74,22 +74,25 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
 		// Postgres cannot handle custom types, resolve to strings
 		switch input.(type) {
 		case *big.Int:
-			var b *big.Int
-			b = input.(*big.Int)
+			b := input.(*big.Int)
 			strValues[fieldName] = b.String()
 		case common.Address:
-			var a common.Address
-			a = input.(common.Address)
+			a := input.(common.Address)
 			strValues[fieldName] = a.String()
 			c.ContractInfo.AddTokenHolderAddress(a.String()) // cache address in a list of contract's token holder addresses
 		case common.Hash:
-			var h common.Hash
-			h = input.(common.Hash)
+			h := input.(common.Hash)
 			strValues[fieldName] = h.String()
 		case string:
 			strValues[fieldName] = input.(string)
 		case bool:
 			strValues[fieldName] = strconv.FormatBool(input.(bool))
+		case []byte:
+			b := input.([]byte)
+			strValues[fieldName] = string(b)
+		case byte:
+			b := input.(byte)
+			strValues[fieldName] = string(b)
 		default:
 			return nil, errors.New(fmt.Sprintf("error: unhandled abi type %T", input))
 		}
@@ -98,7 +101,6 @@ func (c *converter) Convert(watchedEvent core.WatchedEvent, event types.Event) (
 	// Only hold onto logs that pass our address filter, if any
 	if c.ContractInfo.PassesEventFilter(strValues) {
 		eventLog := &types.Log{
-			Event:  event,
 			Id:     watchedEvent.LogID,
 			Values: strValues,
 			Block:  watchedEvent.BlockNumber,

pkg/omni/full/converter/converter_test.go

@@ -25,40 +25,41 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/contract"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers"
 	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers"
+	"github.com/vulcanize/vulcanizedb/pkg/omni/shared/helpers/test_helpers/mocks"
 )
 
 var _ = Describe("Converter", func() {
-	var info *contract.Contract
+	var con *contract.Contract
 	var wantedEvents = []string{"Transfer"}
 	var err error
 
 	BeforeEach(func() {
-		info = test_helpers.SetupTusdContract(wantedEvents, []string{})
+		con = test_helpers.SetupTusdContract(wantedEvents, []string{})
 	})
 
 	Describe("Update", func() {
-		It("Updates contract info held by the converter", func() {
-			c := converter.NewConverter(info)
-			Expect(c.ContractInfo).To(Equal(info))
+		It("Updates contract con held by the converter", func() {
+			c := converter.NewConverter(con)
+			Expect(c.ContractInfo).To(Equal(con))
 
-			info := test_helpers.SetupTusdContract([]string{}, []string{})
-			c.Update(info)
-			Expect(c.ContractInfo).To(Equal(info))
+			con := test_helpers.SetupTusdContract([]string{}, []string{})
+			c.Update(con)
+			Expect(c.ContractInfo).To(Equal(con))
 		})
 	})
 
 	Describe("Convert", func() {
 		It("Converts a watched event log to mapping of event input names to values", func() {
-			_, ok := info.Events["Approval"]
+			_, ok := con.Events["Approval"]
 			Expect(ok).To(Equal(false))
 
-			event, ok := info.Events["Transfer"]
+			event, ok := con.Events["Transfer"]
 			Expect(ok).To(Equal(true))
-			err = info.GenerateFilters()
+			err = con.GenerateFilters()
 			Expect(err).ToNot(HaveOccurred())
 
-			c := converter.NewConverter(info)
-			log, err := c.Convert(test_helpers.MockTranferEvent, event)
+			c := converter.NewConverter(con)
+			log, err := c.Convert(mocks.MockTranferEvent, event)
 			Expect(err).ToNot(HaveOccurred())
 
 			from := common.HexToAddress("0x000000000000000000000000000000000000000000000000000000000000af21")
@@ -72,10 +73,36 @@ var _ = Describe("Converter", func() {
 			Expect(v).To(Equal(value.String()))
 		})
 
+		It("Keeps track of addresses it sees to grow a token holder address list for the contract", func() {
+			event, ok := con.Events["Transfer"]
+			Expect(ok).To(Equal(true))
+
+			c := converter.NewConverter(con)
+			_, err := c.Convert(mocks.MockTranferEvent, event)
+			Expect(err).ToNot(HaveOccurred())
+
+			b, ok := con.TknHolderAddrs["0x000000000000000000000000000000000000Af21"]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			b, ok = con.TknHolderAddrs["0x09BbBBE21a5975cAc061D82f7b843bCE061BA391"]
+			Expect(ok).To(Equal(true))
+			Expect(b).To(Equal(true))
+
+			_, ok = con.TknHolderAddrs["0x"]
+			Expect(ok).To(Equal(false))
+
+			_, ok = con.TknHolderAddrs[""]
+			Expect(ok).To(Equal(false))
+
+			_, ok = con.TknHolderAddrs["0x09THISE21a5IS5cFAKE1D82fAND43bCE06MADEUP"]
+			Expect(ok).To(Equal(false))
+		})
+
 		It("Fails with an empty contract", func() {
-			event := info.Events["Transfer"]
+			event := con.Events["Transfer"]
 			c := converter.NewConverter(&contract.Contract{})
-			_, err = c.Convert(test_helpers.MockTranferEvent, event)
+			_, err = c.Convert(mocks.MockTranferEvent, event)
 			Expect(err).To(HaveOccurred())
 		})
 	})