Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

nimbus-verified-proxy integration with status-go #1497

Closed
wants to merge 1 commit into from
Closed

nimbus-verified-proxy integration with status-go #1497

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitmodules
View file
Open in desktop
Expand Up @@ -65,7 +65,7 @@
branch = master
[submodule "vendor/nim-confutils"]
path = vendor/nim-confutils
url = https://github.com/status-im/nim-confutils.git
url = https://github.com/siphiuel/nim-confutils.git
ignore = dirty
branch = master
[submodule "vendor/nim-blscurve"]
Expand Down
63 changes: 62 additions & 1 deletion Makefile
View file
Open in desktop
Expand Up @@ -70,6 +70,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
nimbus \
fluffy \
nimbus_verified_proxy \
libverifproxy \
test \
test-reproducibility \
clean \
Expand Down Expand Up @@ -272,9 +273,69 @@ evmstate_test: | build deps evmstate
txparse: | build deps
$(ENV_SCRIPT) nim c $(NIM_PARAMS) "tools/txparse/$@.nim"

# Shared library for verified proxy
OS = $(shell $(CC) -dumpmachine)
ifneq (, $(findstring darwin, $(OS)))
SHAREDLIBEXT = a
else
ifneq (, $(findstring mingw, $(OS))$(findstring cygwin, $(OS))$(findstring msys, $(OS)))
SHAREDLIBEXT = lib
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

.lib / .a are static libraries, not shared - ie shared libraries use .dll / .so - which is intended here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, thanks for spotting, will fix. This is a partial carryover from earlier implementation that relied only on dynamic libs.

else
SHAREDLIBEXT = a
endif
endif

VERIF_PROXY_OUT_PATH ?= build/libverifproxy/
VERIFPROXY_OBJS = $(shell find nimcache/libverifproxy -name "*.o")
LIBNATPMP_OBJS = $(shell find vendor/nim-nat-traversal/vendor/libnatpmp-upstream -name "*.o")
LIBMINIUPNPC_OBJS = $(shell find vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc -name "*.o")
LIBBACKTRACE_OBJS = $(shell find vendor/nim-libbacktrace/vendor/libbacktrace-upstream -name "*.o")
LIBBACKTRACE_WRAPPER_OBJS = vendor/nim-libbacktrace/libbacktrace_wrapper.o
ALL_OBJS = $(VERIFPROXY_OBJS) $(LIBNATPMP_OBJS) $(LIBMINIUPNPC_OBJS) $(LIBBACKTRACE_WRAPPER_OBJS) $(LIBBACKTRACE_OBJS)

libverifproxy-objs: | build deps
+ echo -e $(BUILD_MSG) "build/$@" && \
$(ENV_SCRIPT) nim c --noLinking:on --gc:boehm -d:"libp2p_pki_schemes=secp256k1" --header:verifproxy.h --noMain:on --nimcache:nimcache/libverifproxy -o:build/$@ $(NIM_PARAMS) nimbus_verified_proxy/nimbus_verified_proxy.nim && \
mkdir -p build/libverifproxy && \
find nimcache/libverifproxy -name "*.o" | xargs -I {} cp {} $(VERIF_PROXY_OUT_PATH) && \
find vendor/nim-nat-traversal/vendor/libnatpmp-upstream -name "*.o" | xargs -I {} cp {} $(VERIF_PROXY_OUT_PATH) && \
find vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc -name "*.o" | xargs -I {} cp {} $(VERIF_PROXY_OUT_PATH) && \
find vendor/nim-libbacktrace/vendor/libbacktrace-upstream -name "*.o" | xargs -I {} cp {} $(VERIF_PROXY_OUT_PATH) && \
cp vendor/nim-libbacktrace/libbacktrace_wrapper.o $(VERIF_PROXY_OUT_PATH) && \
cp nimcache/libverifproxy/verifproxy.h $(VERIF_PROXY_OUT_PATH)
# echo ${ALL_OBJS} | tr ' ' '\n' > build/libverifproxy-objs/objs.lst
echo -e $(BUILD_END_MSG) "build/$@"




# gc:markAndSweep: exception "URL hostname is missing"
# vendor/nim-json-rpc/json_rpc/rpcproxy.nim(99) start
# vendor/nim-chronos/chronos/asyncfutures2.nim(369) futureContinue

# gc:refc: SIGSEGV

# gc:boehm: ok

# gc:go: Error: system module needs: unsureAsgnRef during compilation
libverifproxy: | build deps
+ echo -e $(BUILD_MSG) "build/$@" && \
$(ENV_SCRIPT) nim --version && \
$(ENV_SCRIPT) nim c --app:staticLib -d:"libp2p_pki_schemes=secp256k1" --opt:size --gc:boehm --header:verifproxy.h --noMain:on --nimcache:nimcache/libverifproxy -o:$(VERIF_PROXY_OUT_PATH)/$@.$(SHAREDLIBEXT) $(NIM_PARAMS) nimbus_verified_proxy/nimbus_verified_proxy.nim
cp nimcache/libverifproxy/verifproxy.h $(VERIF_PROXY_OUT_PATH)/
cp vendor/nimbus-build-system/vendor/Nim-csources-v1/c_code/nimbase.h $(VERIF_PROXY_OUT_PATH)/
echo -e $(BUILD_END_MSG) "build/$@"



libverifproxy-source: | build deps
+ echo -e $(BUILD_MSG) "build/$@" && \
$(ENV_SCRIPT) nim c -c --genScript:on --noLinking:on -d:"libp2p_pki_schemes=secp256k1" --header:verifproxy.h --noMain:on --nimcache:nimcache/libverifproxy -o:build/$@ $(NIM_PARAMS) nimbus_verified_proxy/nimbus_verified_proxy.nim
echo -e $(BUILD_END_MSG) "build/$@"

# usual cleaning
clean: | clean-common
rm -rf build/{nimbus,fluffy,nimbus_verified_proxy,$(TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_fluffy_tests,all_fluffy_portal_spec_tests,test_portal_testnet,portalcli,blockwalk,eth_data_exporter,utp_test_app,utp_test,*.dSYM}
rm -rf build/{nimbus,fluffy,libverifproxy,nimbus_verified_proxy,$(TOOLS_CSV),all_tests,test_kvstore_rocksdb,test_rpc,all_fluffy_tests,all_fluffy_portal_spec_tests,test_portal_testnet,portalcli,blockwalk,eth_data_exporter,utp_test_app,utp_test,*.dSYM}
rm -rf tools/t8n/{t8n,t8n_test}
rm -rf tools/evmstate/{evmstate,evmstate_test}
ifneq ($(USE_LIBBACKTRACE), 0)
Expand Down
127 changes: 118 additions & 9 deletions nimbus_verified_proxy/nimbus_verified_proxy.nim
View file
Open in desktop
Expand Up @@ -8,7 +8,7 @@
{.push raises: [].}

import
std/[os, strutils],
std/[json, os, strutils],
chronicles, chronicles/chronos_tools, chronos, confutils,
eth/keys,
json_rpc/rpcproxy,
Expand Down Expand Up @@ -40,14 +40,29 @@ func getConfiguredChainId(networkMetadata: Eth2NetworkMetadata): Quantity =
else:
return networkMetadata.cfg.DEPOSIT_CHAIN_ID.Quantity

proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
type OnHeaderCallback* = proc (s: cstring) {.cdecl.}

var optimisticHeaderCallback : OnHeaderCallback = nil
var finalizedHeaderCallback : OnHeaderCallback = nil
proc setOptimisticHeaderCallback*(cb: OnHeaderCallback) {.exportc.} =
optimisticHeaderCallback = cb
echo "optimistic header callback set"

proc setFinalizedHeaderCallback*(cb: OnHeaderCallback) {.exportc.} =
finalizedHeaderCallback = cb
echo "finalized header callback set"


proc run(config: VerifiedProxyConf) {.raises: [CatchableError, Exception].} =
# echo "startLightClient inside nimbus-light-client"

# Required as both Eth2Node and LightClient requires correct config type
var lcConfig = config.asLightClientConf()

setupLogging(config.logLevel, config.logStdout, none(OutFile))

notice "Launching Nimbus verified proxy",
version = fullVersionStr, cmdParams = commandLineParams(), config
version = fullVersionStr, cmdParams = getCLIParams(), config

let
metadata = loadEth2Network(config.eth2Network)
Expand Down Expand Up @@ -153,6 +168,14 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
when lcDataFork > LightClientDataFork.None:
info "New LC finalized header",
finalized_header = shortLog(forkyHeader)
if finalizedHeaderCallback != nil:
notice "### Invoking finalizedHeaderCallback"
{.gcsafe.}:
try:
finalizedHeaderCallback(Json.encode(finalizedHeader))
except Exception as e:
notice "finalizedHeaderCallback exception"


proc onOptimisticHeader(
lightClient: LightClient, optimisticHeader: ForkedLightClientHeader) =
Expand All @@ -161,6 +184,14 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
info "New LC optimistic header",
optimistic_header = shortLog(forkyHeader)
optimisticProcessor.setOptimisticHeader(forkyHeader.beacon)
if optimisticHeaderCallback != nil:
notice "### Invoking optimisticHeaderCallback"
{.gcsafe.}:
try:
optimisticHeaderCallback(Json.encode(optimisticHeader))
except Exception:
notice "optimisticHeaderCallback exception"


lightClient.onFinalizedHeader = onFinalizedHeader
lightClient.onOptimisticHeader = onOptimisticHeader
Expand Down Expand Up @@ -239,10 +270,88 @@ proc run(config: VerifiedProxyConf) {.raises: [CatchableError].} =
while true:
poll()

when isMainModule:
{.pop.}
var config = makeBannerAndConfig(
"Nimbus verified proxy " & fullVersionStr, VerifiedProxyConf)
{.push raises: [].}
proc testEcho*() {.exportc.} =
echo "in testEcho"

proc quit*() {.exportc.} =
echo "Quitting"

# template createConfig(clientId: string, ConfType: type, configFilePath: string): untyped =
# echo "### inside createConfig"
# let
# version = clientId & "\p" & copyrights & "\p\p" &
# "eth2 specification v" & SPEC_VERSION & "\p\p" &
# nimBanner

# # TODO for some reason, copyrights are printed when doing `--help`
# {.push warning[ProveInit]: off.}
# let config = try:
# echo "### inside createConfig before load"
# ConfType.load(
# version = version, # but a short version string makes more sense...
# copyrightBanner = clientId,
# secondarySources = proc (config: ConfType, sources: auto) =
# sources.addConfigFile(Toml, InputFile(configFilePath))
# )
# except CatchableError as err:
# # We need to log to stderr here, because logging hasn't been configured yet
# stderr.write "Failure while loading the configuration:\n"
# stderr.write err.msg
# stderr.write "\n"

# if err[] of ConfigurationError and
# err.parent != nil and
# err.parent[] of TomlFieldReadingError:
# let fieldName = ((ref TomlFieldReadingError)(err.parent)).field
# if fieldName in ["web3-url", "bootstrap-node",
# "direct-peer", "validator-monitor-pubkey"]:
# stderr.write "Since the '" & fieldName & "' option is allowed to " &
# "have more than one value, please make sure to supply " &
# "a properly formatted TOML array\n"
# quit 1
# {.pop.}
# config

proc NimMain() {.importc.}
proc startProxyViaJson*(configJson: cstring) {.exportc.} =
echo "startLcViaJson"
NimMain()
echo "startLcViaJson 1"
let str = $configJson
echo "startLcViaJson 2"
echo "startLcViaJson 3 ", str
try:
let jsonNode = parseJson(str)

let rpcAddr = jsonNode["RpcAddress"].getStr()
let config = VerifiedProxyConf(
rpcAddress: ValidIpAddress.init(rpcAddr),
listenAddress: defaultListenAddress,
eth2Network: some(jsonNode["Eth2Network"].getStr()),
trustedBlockRoot: Eth2Digest.fromHex(jsonNode["TrustedBlockRoot"].getStr()),
web3Url: parseCmdArg(ValidatedWeb3Url, jsonNode["Web3Url"].getStr()),
rpcPort: Port(jsonNode["RpcPort"].getInt()),
logLevel: jsonNode["LogLevel"].getStr(),
maxPeers: 160,
nat: NatConfig(hasExtIp: false, nat: NatAny),
logStdout: StdoutLogKind.Auto,
dataDir: OutDir(defaultVerifiedProxyDataDir()),
tcpPort: Port(defaultEth2TcpPort),
udpPort: Port(defaultEth2TcpPort),
agentString: "nimbus",
discv5Enabled: true,
)

run(config)
except Exception as err:
echo "Exception when running ", getCurrentExceptionMsg(), err.getStackTrace()

# when isMainModule:
# let configFileStr = "config.toml"
# {.pop.}
# var config = createConfig("Nimbus verified proxy " & fullVersionStr, VerifiedProxyConf, configFileStr)

# {.push raises: [Defect].}

run(config)
# echo "inside nimbus-light-client before run"
# run(config)
2 changes: 1 addition & 1 deletion vendor/nim-confutils
Open in desktop
Submodule nim-confutils updated from 38dfea to 1cd27f