From 83a30f0945a8fa978f1ae02305c00a0af48e7284 Mon Sep 17 00:00:00 2001 From: Nattharat Wiriyakulnan Date: Mon, 24 Aug 2020 14:58:17 +0700 Subject: [PATCH 1/3] cdb: implement `data_source_requests` table --- chain/emitter/app.go | 2 +- chain/emitter/oracle.go | 13 ++++++++++++- flusher/flusher/db.py | 7 +++++++ flusher/flusher/handler.py | 17 +++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/chain/emitter/app.go b/chain/emitter/app.go index b571002821..d95bf6fd83 100644 --- a/chain/emitter/app.go +++ b/chain/emitter/app.go @@ -160,7 +160,7 @@ func (app *App) InitChain(req abci.RequestInitChain) abci.ResponseInitChain { var oracleState oracle.GenesisState app.Codec().MustUnmarshalJSON(genesisState[oracle.ModuleName], &oracleState) for idx, ds := range oracleState.DataSources { - app.emitSetDataSource(types.DataSourceID(idx+1), ds, nil) + app.emitNewDataSource(types.DataSourceID(idx+1), ds, nil) } for idx, os := range oracleState.OracleScripts { app.emitNewOracleScript(types.OracleScriptID(idx+1), os, nil) diff --git a/chain/emitter/oracle.go b/chain/emitter/oracle.go index f7103aa5ef..a42ade0161 100644 --- a/chain/emitter/oracle.go +++ b/chain/emitter/oracle.go @@ -14,6 +14,17 @@ func parseBytes(b []byte) []byte { return b } +func (app *App) emitNewDataSource(id types.DataSourceID, ds types.DataSource, txHash []byte) { + app.Write("NEW_DATA_SOURCE", JsDict{ + "id": id, + "name": ds.Name, + "description": ds.Description, + "owner": ds.Owner.String(), + "executable": app.OracleKeeper.GetFile(ds.Filename), + "tx_hash": txHash, + }) +} + func (app *App) emitSetDataSource(id types.DataSourceID, ds types.DataSource, txHash []byte) { app.Write("SET_DATA_SOURCE", JsDict{ "id": id, @@ -125,7 +136,7 @@ func (app *App) handleMsgCreateDataSource( ) { id := types.DataSourceID(atoi(evMap[types.EventTypeCreateDataSource+"."+types.AttributeKeyID][0])) ds := app.BandApp.OracleKeeper.MustGetDataSource(app.DeliverContext, id) - app.emitSetDataSource(id, ds, txHash) + app.emitNewDataSource(id, ds, txHash) extra["id"] = id } diff --git a/flusher/flusher/db.py b/flusher/flusher/db.py index 50aa88931a..937c26508a 100644 --- a/flusher/flusher/db.py +++ b/flusher/flusher/db.py @@ -360,6 +360,13 @@ def Column(*args, **kwargs): Column("timestamp", CustomDateTime, primary_key=True), ) +data_source_requests = sa.Table( + "data_source_requests", + metadata, + Column("data_source_id", sa.Integer, sa.ForeignKey("data_sources.id"), primary_key=True), + Column("count", sa.Integer), +) + oracle_script_requests = sa.Table( "oracle_script_requests", metadata, diff --git a/flusher/flusher/handler.py b/flusher/flusher/handler.py index e27165d8c6..dca1ccd0ec 100644 --- a/flusher/flusher/handler.py +++ b/flusher/flusher/handler.py @@ -27,6 +27,7 @@ reporters, related_data_source_oracle_scripts, historical_oracle_statuses, + data_source_requests, oracle_script_requests, request_count_per_days, ) @@ -79,6 +80,10 @@ def handle_set_account(self, msg): condition = (col == msg[col.name]) & condition self.conn.execute(accounts.update().where(condition).values(**msg)) + def handle_new_data_source(self, msg): + self.handle_set_data_source(msg) + self.handle_new_data_source_request({"data_source_id": msg["id"], "count": 0}) + def handle_set_data_source(self, msg): if msg["tx_hash"] is not None: msg["transaction_id"] = self.get_transaction_id(msg["tx_hash"]) @@ -138,6 +143,7 @@ def handle_new_raw_request(self, msg): } ) self.conn.execute(raw_requests.insert(), msg) + self.handle_set_data_source_request({"data_source_id": msg["data_source_id"]}) def handle_new_val_request(self, msg): msg["validator_id"] = self.get_validator_id(msg["validator"]) @@ -309,6 +315,17 @@ def handle_set_historical_validator_status(self, msg): .on_conflict_do_update(constraint="historical_oracle_statuses_pkey", set_=msg) ) + def handle_new_data_source_request(self, msg): + self.conn.execute(data_source_requests.insert(), msg) + + def handle_set_data_source_request(self, msg): + condition = True + for col in data_source_requests.primary_key.columns.values(): + condition = (col == msg[col.name]) & condition + self.conn.execute( + data_source_requests.update(condition).values(count=data_source_requests.c.count + 1) + ) + def handle_new_oracle_script_request(self, msg): self.conn.execute(oracle_script_requests.insert(), msg) From 4f9ec64d20bdfe3d4cd6671021443c9c7c582463 Mon Sep 17 00:00:00 2001 From: Nattharat Wiriyakulnan Date: Mon, 24 Aug 2020 15:01:08 +0700 Subject: [PATCH 2/3] update change log --- CHANGELOG_UNRELEASED.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG_UNRELEASED.md b/CHANGELOG_UNRELEASED.md index 238b478531..028e990ba0 100644 --- a/CHANGELOG_UNRELEASED.md +++ b/CHANGELOG_UNRELEASED.md @@ -16,6 +16,7 @@ ### Emitter & Flusher +- (impv) [\#2549](https://github.com/bandprotocol/bandchain/pull/2549) Implemented `data_source_requests` table - (feat) [\#2551](https://github.com/bandprotocol/bandchain/pull/2551) fast-sync: add flag enable fast sync and emit all account and validator ### Scan @@ -23,7 +24,7 @@ - (impv) [\#2557](https://github.com/bandprotocol/bandchain/pull/2557) Fix and clean up copy button - (impv) [\#2552](https://github.com/bandprotocol/bandchain/pull/2552) Wire up related data source, handle nullable timestamp - (impv) [\#2550](https://github.com/bandprotocol/bandchain/pull/2550) Polish request index page -- (bugs) [\#2527](https://github.com/bandprotocol/bandchain/pull/2527) Fix serach bugs on ds & os home page +- (bugs) [\#2527](https://github.com/bandprotocol/bandchain/pull/2527) Fix search bugs on ds & os home page - (impv) [\#2516](https://github.com/bandprotocol/bandchain/pull/2516) Fix loading width and bg color - (impv) [\#2499](https://github.com/bandprotocol/bandchain/pull/2499) Implemented Total request chart - (impv) [\#2455](https://github.com/bandprotocol/bandchain/pull/2455) Implement full copy non evm proof From 17b3b582ced698a15278924efa491e9aac325614 Mon Sep 17 00:00:00 2001 From: Nattharat Wiriyakulnan Date: Tue, 25 Aug 2020 13:13:35 +0700 Subject: [PATCH 3/3] fix as comment --- chain/emitter/app.go | 4 ++-- chain/emitter/oracle.go | 28 ++-------------------------- flusher/flusher/handler.py | 24 ++++++++++-------------- 3 files changed, 14 insertions(+), 42 deletions(-) diff --git a/chain/emitter/app.go b/chain/emitter/app.go index d95bf6fd83..acfd852324 100644 --- a/chain/emitter/app.go +++ b/chain/emitter/app.go @@ -160,10 +160,10 @@ func (app *App) InitChain(req abci.RequestInitChain) abci.ResponseInitChain { var oracleState oracle.GenesisState app.Codec().MustUnmarshalJSON(genesisState[oracle.ModuleName], &oracleState) for idx, ds := range oracleState.DataSources { - app.emitNewDataSource(types.DataSourceID(idx+1), ds, nil) + app.emitSetDataSource(types.DataSourceID(idx+1), ds, nil) } for idx, os := range oracleState.OracleScripts { - app.emitNewOracleScript(types.OracleScriptID(idx+1), os, nil) + app.emitSetOracleScript(types.OracleScriptID(idx+1), os, nil) } app.FlushMessages() return res diff --git a/chain/emitter/oracle.go b/chain/emitter/oracle.go index a42ade0161..da47d0d364 100644 --- a/chain/emitter/oracle.go +++ b/chain/emitter/oracle.go @@ -14,17 +14,6 @@ func parseBytes(b []byte) []byte { return b } -func (app *App) emitNewDataSource(id types.DataSourceID, ds types.DataSource, txHash []byte) { - app.Write("NEW_DATA_SOURCE", JsDict{ - "id": id, - "name": ds.Name, - "description": ds.Description, - "owner": ds.Owner.String(), - "executable": app.OracleKeeper.GetFile(ds.Filename), - "tx_hash": txHash, - }) -} - func (app *App) emitSetDataSource(id types.DataSourceID, ds types.DataSource, txHash []byte) { app.Write("SET_DATA_SOURCE", JsDict{ "id": id, @@ -36,19 +25,6 @@ func (app *App) emitSetDataSource(id types.DataSourceID, ds types.DataSource, tx }) } -func (app *App) emitNewOracleScript(id types.OracleScriptID, os types.OracleScript, txHash []byte) { - app.Write("NEW_ORACLE_SCRIPT", JsDict{ - "id": id, - "name": os.Name, - "description": os.Description, - "owner": os.Owner.String(), - "schema": os.Schema, - "codehash": os.Filename, - "source_code_url": os.SourceCodeURL, - "tx_hash": txHash, - }) -} - func (app *App) emitSetOracleScript(id types.OracleScriptID, os types.OracleScript, txHash []byte) { app.Write("SET_ORACLE_SCRIPT", JsDict{ "id": id, @@ -136,7 +112,7 @@ func (app *App) handleMsgCreateDataSource( ) { id := types.DataSourceID(atoi(evMap[types.EventTypeCreateDataSource+"."+types.AttributeKeyID][0])) ds := app.BandApp.OracleKeeper.MustGetDataSource(app.DeliverContext, id) - app.emitNewDataSource(id, ds, txHash) + app.emitSetDataSource(id, ds, txHash) extra["id"] = id } @@ -146,7 +122,7 @@ func (app *App) handleMsgCreateOracleScript( ) { id := types.OracleScriptID(atoi(evMap[types.EventTypeCreateOracleScript+"."+types.AttributeKeyID][0])) os := app.BandApp.OracleKeeper.MustGetOracleScript(app.DeliverContext, id) - app.emitNewOracleScript(id, os, txHash) + app.emitSetOracleScript(id, os, txHash) extra["id"] = id } diff --git a/flusher/flusher/handler.py b/flusher/flusher/handler.py index dca1ccd0ec..3998fdf175 100644 --- a/flusher/flusher/handler.py +++ b/flusher/flusher/handler.py @@ -80,10 +80,6 @@ def handle_set_account(self, msg): condition = (col == msg[col.name]) & condition self.conn.execute(accounts.update().where(condition).values(**msg)) - def handle_new_data_source(self, msg): - self.handle_set_data_source(msg) - self.handle_new_data_source_request({"data_source_id": msg["id"], "count": 0}) - def handle_set_data_source(self, msg): if msg["tx_hash"] is not None: msg["transaction_id"] = self.get_transaction_id(msg["tx_hash"]) @@ -95,10 +91,11 @@ def handle_set_data_source(self, msg): .values(**msg) .on_conflict_do_update(constraint="data_sources_pkey", set_=msg) ) - - def handle_new_oracle_script(self, msg): - self.handle_set_oracle_script(msg) - self.handle_new_oracle_script_request({"oracle_script_id": msg["id"], "count": 0}) + self.conn.execute( + insert(data_source_requests) + .values({"data_source_id": msg["id"], "count": 0}) + .on_conflict_do_nothing(constraint="data_source_requests_pkey") + ) def handle_set_oracle_script(self, msg): if msg["tx_hash"] is not None: @@ -111,6 +108,11 @@ def handle_set_oracle_script(self, msg): .values(**msg) .on_conflict_do_update(constraint="oracle_scripts_pkey", set_=msg) ) + self.conn.execute( + insert(oracle_script_requests) + .values({"oracle_script_id": msg["id"], "count": 0}) + .on_conflict_do_nothing(constraint="oracle_script_requests_pkey") + ) def handle_new_request(self, msg): msg["transaction_id"] = self.get_transaction_id(msg["tx_hash"]) @@ -315,9 +317,6 @@ def handle_set_historical_validator_status(self, msg): .on_conflict_do_update(constraint="historical_oracle_statuses_pkey", set_=msg) ) - def handle_new_data_source_request(self, msg): - self.conn.execute(data_source_requests.insert(), msg) - def handle_set_data_source_request(self, msg): condition = True for col in data_source_requests.primary_key.columns.values(): @@ -326,9 +325,6 @@ def handle_set_data_source_request(self, msg): data_source_requests.update(condition).values(count=data_source_requests.c.count + 1) ) - def handle_new_oracle_script_request(self, msg): - self.conn.execute(oracle_script_requests.insert(), msg) - def handle_set_oracle_script_request(self, msg): condition = True for col in oracle_script_requests.primary_key.columns.values():