(VDB-366) Add sin mapping to vow storage transformer

db/migrations/20181114200132_create_vow_storage_tables.sql

@@ -20,13 +20,21 @@ CREATE TABLE maker.vow_row (
   row           TEXT
 );
 
-CREATE TABLE maker.vow_sin (
+CREATE TABLE maker.vow_sin_integer (
   id            SERIAL PRIMARY KEY,
   block_number  BIGINT,
   block_hash    TEXT,
   sin           numeric
 );
 
+CREATE TABLE maker.vow_sin_mapping (
+  id            SERIAL PRIMARY KEY,
+  block_number  BIGINT,
+  block_hash    TEXT,
+  timestamp     numeric,
+  sin           numeric
+);
+
 CREATE TABLE maker.vow_ash (
   id            SERIAL PRIMARY KEY,
   block_number  BIGINT,

db/schema.sql

@@ -2996,10 +2996,10 @@ ALTER SEQUENCE maker.vow_row_id_seq OWNED BY maker.vow_row.id;
 
 
 --
--- Name: vow_sin; Type: TABLE; Schema: maker; Owner: -
+-- Name: vow_sin_integer; Type: TABLE; Schema: maker; Owner: -
 --
 
-CREATE TABLE maker.vow_sin (
+CREATE TABLE maker.vow_sin_integer (
     id integer NOT NULL,
     block_number bigint,
     block_hash text,
@@ -3008,10 +3008,10 @@ CREATE TABLE maker.vow_sin (
 
 
 --
--- Name: vow_sin_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+-- Name: vow_sin_integer_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
 --
 
-CREATE SEQUENCE maker.vow_sin_id_seq
+CREATE SEQUENCE maker.vow_sin_integer_id_seq
     AS integer
     START WITH 1
     INCREMENT BY 1
@@ -3021,10 +3021,43 @@ CREATE SEQUENCE maker.vow_sin_id_seq
 
 
 --
--- Name: vow_sin_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+-- Name: vow_sin_integer_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
 --
 
-ALTER SEQUENCE maker.vow_sin_id_seq OWNED BY maker.vow_sin.id;
+ALTER SEQUENCE maker.vow_sin_integer_id_seq OWNED BY maker.vow_sin_integer.id;
+
+
+--
+-- Name: vow_sin_mapping; Type: TABLE; Schema: maker; Owner: -
+--
+
+CREATE TABLE maker.vow_sin_mapping (
+    id integer NOT NULL,
+    block_number bigint,
+    block_hash text,
+    "timestamp" numeric,
+    sin numeric
+);
+
+
+--
+-- Name: vow_sin_mapping_id_seq; Type: SEQUENCE; Schema: maker; Owner: -
+--
+
+CREATE SEQUENCE maker.vow_sin_mapping_id_seq
+    AS integer
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+
+--
+-- Name: vow_sin_mapping_id_seq; Type: SEQUENCE OWNED BY; Schema: maker; Owner: -
+--
+
+ALTER SEQUENCE maker.vow_sin_mapping_id_seq OWNED BY maker.vow_sin_mapping.id;
 
 
 --
@@ -4150,10 +4183,17 @@ ALTER TABLE ONLY maker.vow_row ALTER COLUMN id SET DEFAULT nextval('maker.vow_ro
 
 
 --
--- Name: vow_sin id; Type: DEFAULT; Schema: maker; Owner: -
+-- Name: vow_sin_integer id; Type: DEFAULT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vow_sin_integer ALTER COLUMN id SET DEFAULT nextval('maker.vow_sin_integer_id_seq'::regclass);
+
+
+--
+-- Name: vow_sin_mapping id; Type: DEFAULT; Schema: maker; Owner: -
 --
 
-ALTER TABLE ONLY maker.vow_sin ALTER COLUMN id SET DEFAULT nextval('maker.vow_sin_id_seq'::regclass);
+ALTER TABLE ONLY maker.vow_sin_mapping ALTER COLUMN id SET DEFAULT nextval('maker.vow_sin_mapping_id_seq'::regclass);
 
 
 --
@@ -5028,11 +5068,19 @@ ALTER TABLE ONLY maker.vow_row
 
 
 --
--- Name: vow_sin vow_sin_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+-- Name: vow_sin_integer vow_sin_integer_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
+--
+
+ALTER TABLE ONLY maker.vow_sin_integer
+    ADD CONSTRAINT vow_sin_integer_pkey PRIMARY KEY (id);
+
+
+--
+-- Name: vow_sin_mapping vow_sin_mapping_pkey; Type: CONSTRAINT; Schema: maker; Owner: -
 --
 
-ALTER TABLE ONLY maker.vow_sin
-    ADD CONSTRAINT vow_sin_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY maker.vow_sin_mapping
+    ADD CONSTRAINT vow_sin_mapping_pkey PRIMARY KEY (id);
 
 
 --

test_config/test_config.go

@@ -168,8 +168,17 @@ func CleanTestDB(db *postgres.DB) {
 	db.MustExec("DELETE FROM maker.vat_urn_art")
 	db.MustExec("DELETE FROM maker.vat_urn_ink")
 	db.MustExec("DELETE FROM maker.vat_vice")
+	db.MustExec("DELETE FROM maker.vow_ash")
+	db.MustExec("DELETE FROM maker.vow_bump")
+	db.MustExec("DELETE FROM maker.vow_cow")
 	db.MustExec("DELETE FROM maker.vow_fess")
 	db.MustExec("DELETE FROM maker.vow_flog")
+	db.MustExec("DELETE FROM maker.vow_hump")
+	db.MustExec("DELETE FROM maker.vow_row")
+	db.MustExec("DELETE FROM maker.vow_sin_integer")
+	db.MustExec("DELETE FROM maker.vow_sump")
+	db.MustExec("DELETE FROM maker.vow_vat")
+	db.MustExec("DELETE FROM maker.vow_wait")
 	db.MustExec("DELETE FROM watched_contracts")
 	// TODO: add ON DELETE CASCADE? otherwise these need to come after deleting tables that reference it
 	db.MustExec("DELETE FROM maker.urns")

transformers/shared/constants/keys.go

@@ -19,7 +19,8 @@ package constants
 import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 
 const (
-	Ilk  utils.Key = "ilk"
-	Guy  utils.Key = "guy"
-	Flip utils.Key = "flip"
+	Ilk       utils.Key = "ilk"
+	Guy       utils.Key = "guy"
+	Flip      utils.Key = "flip"
+	Timestamp utils.Key = "timestamp"
 )

transformers/storage/repository.go

@@ -33,7 +33,8 @@ type IMakerStorageRepository interface {
 	GetMaxFlip() (*big.Int, error)
 	GetGemKeys() ([]Urn, error)
 	GetIlks() ([]string, error)
-	GetSinKeys() ([]string, error)
+	GetVatSinKeys() ([]string, error)
+	GetVowSinKeys() ([]string, error)
 	GetUrns() ([]Urn, error)
 	SetDB(db *postgres.DB)
 }
@@ -99,7 +100,7 @@ func (repository MakerStorageRepository) GetIlks() ([]string, error) {
 	return ilks, err
 }
 
-func (repository *MakerStorageRepository) GetSinKeys() ([]string, error) {
+func (repository *MakerStorageRepository) GetVatSinKeys() ([]string, error) {
 	var sinKeys []string
 	err := repository.db.Select(&sinKeys, `
 		SELECT DISTINCT w FROM maker.vat_grab
@@ -108,13 +109,23 @@ func (repository *MakerStorageRepository) GetSinKeys() ([]string, error) {
 	return sinKeys, err
 }
 
+func (repository *MakerStorageRepository) GetVowSinKeys() ([]string, error) {
+	var sinKeys []string
+	err := repository.db.Select(&sinKeys, `
+		SELECT DISTINCT era FROM maker.vow_flog
+		UNION
+		SELECT DISTINCT headers.block_timestamp
+		FROM maker.vow_fess
+		JOIN headers ON maker.vow_fess.header_id = headers.id`)
+	return sinKeys, err
+}
+
 func (repository *MakerStorageRepository) GetUrns() ([]Urn, error) {
 	var urns []Urn
 	err := repository.db.Select(&urns, `
 		SELECT DISTINCT ilks.ilk, urns.guy
 		FROM maker.urns
-		JOIN maker.ilks on maker.ilks.id = maker.urns.ilk_id
-`)
+		JOIN maker.ilks on maker.ilks.id = maker.urns.ilk_id`)
 	return urns, err
 }
 

transformers/storage/repository_test.go

@@ -17,8 +17,11 @@
 package storage_test
 
 import (
+	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"math/big"
+	"strconv"
 
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres/repositories"
@@ -38,6 +41,9 @@ var _ = Describe("Maker storage repository", func() {
 		guy1       = "guy1"
 		guy2       = "guy2"
 		guy3       = "guy3"
+		era        = big.NewInt(0).SetBytes(common.FromHex("0x000000000000000000000000000000000000000000000000000000005bb48864")).String()
+		tab        = big.NewInt(0).SetBytes(common.FromHex("0x0000000000000000000000000000000000000000000002544faa778090e00000")).String()
+		timestamp  = int64(1538558053)
 	)
 
 	BeforeEach(func() {
@@ -206,11 +212,11 @@ var _ = Describe("Maker storage repository", func() {
 		})
 	})
 
-	Describe("getting sin keys", func() {
+	Describe("getting vat sin keys", func() {
 		It("fetches guy from w field of vat grab", func() {
 			insertVatGrab(guy1, guy1, guy1, guy2, 1, db)
 
-			sinKeys, err := repository.GetSinKeys()
+			sinKeys, err := repository.GetVatSinKeys()
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(sinKeys)).To(Equal(1))
@@ -220,7 +226,7 @@ var _ = Describe("Maker storage repository", func() {
 		It("fetches guy from u field of vat heal", func() {
 			insertVatHeal(guy1, guy2, 1, db)
 
-			sinKeys, err := repository.GetSinKeys()
+			sinKeys, err := repository.GetVatSinKeys()
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(sinKeys)).To(Equal(1))
@@ -234,15 +240,58 @@ var _ = Describe("Maker storage repository", func() {
 			insertVatGrab(guy2, guy2, guy2, guy2, 3, db)
 			insertVatHeal(guy1, guy2, 4, db)
 
-			sinKeys, err := repository.GetSinKeys()
+			sinKeys, err := repository.GetVatSinKeys()
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(sinKeys)).To(Equal(2))
 			Expect(sinKeys).To(ConsistOf(guy1, guy2))
 		})
 
 		It("does not return error if no matching rows", func() {
-			sinKeys, err := repository.GetSinKeys()
+			sinKeys, err := repository.GetVatSinKeys()
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(sinKeys)).To(BeZero())
+		})
+	})
+
+	Describe("getting vow sin keys", func() {
+		It("fetches timestamp from era field of vow flog", func() {
+			insertVowFlog(era, 1, db)
+
+			sinKeys, err := repository.GetVowSinKeys()
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(sinKeys)).To(Equal(1))
+			Expect(sinKeys).To(ConsistOf(era))
+		})
+
+		It("fetches timestamp from header of vow fess event", func() {
+			insertVowFess(tab, timestamp, 1, db)
+
+			sinKeys, err := repository.GetVowSinKeys()
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(sinKeys)).To(Equal(1))
+			Expect(sinKeys).To(ConsistOf(strconv.FormatInt(timestamp, 10)))
+		})
+
+		It("fetches unique sin keys from vow flog and vow fess header", func() {
+			insertVowFlog(era, 1, db)
+			insertVowFess(tab, timestamp, 2, db)
+			// duplicates
+			insertVowFlog(era, 3, db)
+			insertVowFess(tab, timestamp, 4, db)
+
+			sinKeys, err := repository.GetVowSinKeys()
+
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(sinKeys)).To(Equal(2))
+			Expect(sinKeys).To(ConsistOf(era, strconv.FormatInt(timestamp, 10)))
+		})
+
+		It("does not return error if no matching rows", func() {
+			sinKeys, err := repository.GetVowSinKeys()
 
 			Expect(err).NotTo(HaveOccurred())
 			Expect(len(sinKeys)).To(BeZero())
@@ -331,6 +380,35 @@ func insertVatGrab(ilk, urn, v, w string, blockNumber int64, db *postgres.DB) {
 	Expect(execErr).NotTo(HaveOccurred())
 }
 
+func insertVowFlog(era string, blockNumber int64, db *postgres.DB) {
+	headerRepository := repositories.NewHeaderRepository(db)
+	headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber))
+
+	Expect(err).NotTo(HaveOccurred())
+	_, execErr := db.Exec(
+		`INSERT INTO maker.vow_flog (header_id, era, log_idx, tx_idx)
+			VALUES($1, $2, $3, $4)`,
+		headerID, era, 0, 0,
+	)
+	Expect(execErr).NotTo(HaveOccurred())
+}
+
+func insertVowFess(tab string, timestamp, blockNumber int64, db *postgres.DB) {
+	headerRepository := repositories.NewHeaderRepository(db)
+	fakeHeader := fakes.GetFakeHeader(blockNumber)
+	fakeHeader.Timestamp = strconv.FormatInt(timestamp, 10)
+	// TODO: replace above 2 lines with fakes.GetFakeHeaderWithTimestamp once it's in a versioned release
+	headerID, err := headerRepository.CreateOrUpdateHeader(fakeHeader)
+
+	Expect(err).NotTo(HaveOccurred())
+	_, execErr := db.Exec(
+		`INSERT INTO maker.vow_fess (header_id, tab, log_idx, tx_idx)
+			VALUES($1, $2, $3, $4)`,
+		headerID, tab, 0, 0,
+	)
+	Expect(execErr).NotTo(HaveOccurred())
+}
+
 func insertVatHeal(urn, v string, blockNumber int64, db *postgres.DB) {
 	headerRepository := repositories.NewHeaderRepository(db)
 	headerID, err := headerRepository.CreateOrUpdateHeader(fakes.GetFakeHeader(blockNumber))