rework spec imports (#2779)

The spec imports are a mess to work with, so this branch cleans them up
a bit to ensure that we avoid generic sandwitches and that importing
stuff generally becomes easier.

* reexport crypto/digest/presets because these are part of the public
symbol set of the rest of the spec types
* don't export `merge` types from `base` - this causes circular deps
* fix circular deps in `ssz/spec_types` - this is the first step in
disentangling ssz from spec
* be explicit about phase0 vs altair - longer term, `altair` will become
the "natural" type set, then merge and so on, so no point in giving
`phase0` special preferential treatment

beacon_chain/beacon_chain_db.nim

@@ -13,12 +13,12 @@ import
   serialization, chronicles, snappy,
   eth/db/[kvstore, kvstore_sqlite3],
   ./networking/network_metadata, ./beacon_chain_db_immutable,
-  ./spec/[crypto, digest, state_transition],
+  ./spec/state_transition,
   ./spec/datatypes/[phase0, altair],
   ./ssz/[ssz_serialization, merkleization],
   ./filepath
 
-export crypto
+export phase0, altair
 
 logScope: topics = "bc_db"
 

beacon_chain/beacon_chain_db_immutable.nim

@@ -11,7 +11,6 @@ import
   stew/[assign2, objects, results],
   serialization,
   eth/db/kvstore,
-  ./spec/[crypto, digest],
   ./spec/datatypes/[base, altair],
   ./ssz/[ssz_serialization, merkleization]
 

beacon_chain/beacon_clock.nim

@@ -9,7 +9,7 @@
 
 import
   chronos, chronicles,
-  ./spec/[datatypes, helpers]
+  ./spec/helpers
 
 from times import Time, getTime, fromUnix, `<`, `-`, inNanoseconds
 

beacon_chain/beacon_node_types.nim

@@ -10,8 +10,7 @@
 import
   std/[deques, intsets, streams, tables],
   stew/endians2,
-  ./spec/[digest, crypto],
-  ./spec/datatypes/base,
+  ./spec/datatypes/[phase0, altair],
   ./consensus_object_pools/block_pools_types,
   ./fork_choice/fork_choice_types,
   ./validators/slashing_protection

beacon_chain/conf.nim

@@ -17,7 +17,7 @@ import
   eth/p2p/discoveryv5/enr,
   json_serialization, web3/[ethtypes, confutils_defs],
 
-  ./spec/[crypto, keystore, digest, network],
+  ./spec/[keystore, network],
   ./spec/datatypes/base,
   ./networking/network_metadata,
   ./validators/slashing_protection_common,

beacon_chain/consensus_object_pools/attestation_pool.nim

@@ -14,13 +14,11 @@ import
   metrics,
   chronicles, stew/byteutils, json_serialization/std/sets as jsonSets,
   # Internal
-  ../spec/[
-    beaconstate, crypto, digest, forks,
-    validator],
+  ../spec/[beaconstate, forks, helpers, validator],
   ../spec/datatypes/[phase0, altair],
   ../ssz/[merkleization, types],
   "."/[spec_cache, blockchain_dag, block_quarantine],
-  ".."/[beacon_clock, beacon_node_types, extras],
+  ".."/[beacon_clock, beacon_node_types],
   ../fork_choice/fork_choice
 
 export beacon_node_types

beacon_chain/consensus_object_pools/block_clearance.nim

@@ -12,12 +12,10 @@ import
   chronicles,
   stew/[assign2, results],
   eth/keys,
-  ../extras, ../beacon_clock,
-  ../spec/[
-    crypto, digest, forks, helpers, signatures,
-    signatures_batch, state_transition],
+  ".."/[beacon_clock],
+  ../spec/[forks, helpers, signatures, signatures_batch, state_transition],
   ../spec/datatypes/[phase0, altair],
-  ./block_pools_types, ./blockchain_dag, ./block_quarantine
+  "."/[block_pools_types, blockchain_dag, block_quarantine]
 
 from libp2p/protocols/pubsub/pubsub import ValidationResult
 

beacon_chain/consensus_object_pools/block_pools_types.nim

@@ -14,9 +14,9 @@ import
   stew/endians2, chronicles,
   eth/keys,
   # Internals
-  ../spec/[crypto, digest, signatures_batch, forks],
+  ../spec/[signatures_batch, forks],
   ../spec/datatypes/[phase0, altair],
-  ".."/[beacon_chain_db, beacon_clock, extras]
+  ".."/[beacon_chain_db, beacon_clock]
 
 export sets, tables
 

beacon_chain/consensus_object_pools/block_quarantine.nim

@@ -12,7 +12,6 @@ import
   chronicles,
   stew/bitops2,
   eth/keys,
-  ../spec/[crypto, digest],
   ../spec/datatypes/[phase0, altair],
   ./block_pools_types
 

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -11,15 +11,13 @@ import
   std/[options, sequtils, tables, sets],
   stew/[assign2, byteutils],
   metrics, snappy, chronicles,
-  ../ssz/[ssz_serialization, merkleization], ../beacon_chain_db, ../extras,
-  ../spec/[
-    crypto, digest, helpers, validator, state_transition,
-    beaconstate, forks],
+  ../ssz/[ssz_serialization, merkleization],
+  ../spec/[helpers, validator, state_transition, beaconstate, forks],
   ../spec/datatypes/[phase0, altair],
-  ../beacon_clock,
+  ".."/[beacon_clock, beacon_chain_db],
   "."/[block_pools_types, block_quarantine, forkedbeaconstate_dbhelpers]
 
-export block_pools_types, helpers, phase0
+export block_pools_types
 
 # https://github.com/ethereum/eth2.0-metrics/blob/master/metrics.md#interop-metrics
 declareGauge beacon_head_root, "Root of the head block of the beacon chain"

beacon_chain/consensus_object_pools/exit_pool.nim

@@ -13,7 +13,7 @@ import
   # Status libraries
   chronicles,
   # Internal
-  ../spec/[crypto, forks, helpers],
+  ../spec/[forks, helpers],
   ../spec/datatypes/base,
   "."/[blockchain_dag, block_quarantine],
   ../beacon_node_types

beacon_chain/consensus_object_pools/spec_cache.nim

@@ -10,10 +10,13 @@
 import
   std/[intsets],
   chronicles,
-  ../spec/[crypto, digest, helpers, network, presets, signatures, validator],
-  ../spec/datatypes/base,
   ../extras,
-  ./block_pools_types, ./blockchain_dag
+  ../spec/[helpers, network, signatures, validator],
+  ../spec/datatypes/base,
+  ./block_pools_types, blockchain_dag
+
+export
+  base, extras
 
 # Spec functions implemented based on cached values instead of the full state
 func count_active_validators*(epochInfo: EpochRef): uint64 =

beacon_chain/eth1/deposit_contract.nim

@@ -14,7 +14,7 @@ import
   ../networking/network_metadata,
   web3, web3/confutils_defs, eth/keys, eth/p2p/discoveryv5/random2,
   stew/io2,
-  ../spec/[crypto, presets], ../spec/datatypes/base, ../ssz/merkleization,
+  ../spec/datatypes/base, ../ssz/merkleization,
   ../validators/keystore_management
 
 # Compiled version of /scripts/depositContract.v.py in this repo

beacon_chain/eth1/eth1_monitor.nim

@@ -15,8 +15,8 @@ import
   web3, web3/ethtypes as web3Types, web3/ethhexstrings, eth/common/eth_types,
   eth/async_utils, stew/byteutils,
   # Local modules:
-  ../spec/[digest, crypto, forks, helpers],
-  ../spec/datatypes/base,
+  ../spec/[forks, helpers],
+  ../spec/datatypes/[base, merge],
   ../networking/network_metadata,
   ../consensus_object_pools/block_pools_types,
   ../ssz,

beacon_chain/fork_choice/fork_choice.nim

@@ -13,7 +13,7 @@ import
   # Status libraries
   stew/results, chronicles,
   # Internal
-  ../spec/[beaconstate, digest, helpers],
+  ../spec/[beaconstate, helpers],
   ../spec/datatypes/[phase0, altair],
   # Fork choice
   ./fork_choice_types, ./proto_array,

beacon_chain/fork_choice/fork_choice_types.nim

@@ -16,7 +16,6 @@ import
   chronicles,
   # Internal
   ../spec/datatypes/base,
-  ../spec/digest,
   ../consensus_object_pools/block_pools_types
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/fork-choice.md

beacon_chain/fork_choice/proto_array.nim

@@ -15,7 +15,6 @@ import
   stew/results,
   # Internal
   ../spec/datatypes/base,
-  ../spec/digest,
   # Fork choice
   ./fork_choice_types
 

beacon_chain/gossip_processing/batch_validation.nim

@@ -13,7 +13,7 @@ import
   stew/results,
   eth/keys,
   # Internals
-  ../spec/[crypto, digest, helpers, signatures_batch],
+  ../spec/[helpers, signatures_batch],
   ../spec/datatypes/base,
   ../consensus_object_pools/[
     blockchain_dag, block_quarantine,

beacon_chain/gossip_processing/block_processor.nim

@@ -12,7 +12,7 @@ import
   stew/results,
   chronicles, chronos, metrics,
   ../spec/datatypes/[phase0, altair],
-  ../spec/[crypto, digest, forks],
+  ../spec/[forks],
   ../consensus_object_pools/[block_clearance, blockchain_dag, attestation_pool],
   ./consensus_manager,
   ".."/[beacon_clock, beacon_node_types],

beacon_chain/gossip_processing/consensus_manager.nim

@@ -8,8 +8,8 @@
 {.push raises: [Defect].}
 
 import
-  chronicles, chronos,
-  ../spec/[crypto, datatypes],
+  chronicles,
+  ../spec/datatypes/base,
   ../consensus_object_pools/[blockchain_dag, attestation_pool]
 
 # TODO: Move to "consensus_object_pools" folder

beacon_chain/gossip_processing/eth2_processor.nim

@@ -11,7 +11,7 @@ import
   std/tables,
   stew/results,
   chronicles, chronos, metrics,
-  ../spec/[crypto, digest, forks],
+  ../spec/[helpers, forks],
   ../spec/datatypes/[altair, phase0],
   ../consensus_object_pools/[block_clearance, blockchain_dag, exit_pool, attestation_pool],
   ./gossip_validation, ./block_processor,

beacon_chain/gossip_processing/gossip_validation.nim

@@ -16,16 +16,13 @@ import
   # Internals
   ../spec/datatypes/[phase0, altair],
   ../spec/[
-    beaconstate, state_transition_block,
-    crypto, digest, forks, helpers, network,
-    signatures],
+    beaconstate, state_transition_block, forks, helpers, network, signatures],
   ../consensus_object_pools/[
     spec_cache, blockchain_dag, block_quarantine, spec_cache,
     attestation_pool, exit_pool
   ],
   ".."/[beacon_node_types, ssz, beacon_clock],
   ../validators/attestation_aggregation,
-  ../extras,
   ./batch_validation
 
 from libp2p/protocols/pubsub/pubsub import ValidationResult

beacon_chain/interop.nim

@@ -9,8 +9,9 @@
 
 import
   stew/endians2, stint,
-  ./extras, ./ssz/merkleization,
-  spec/[crypto, digest, keystore, signatures],
+  ./ssz/merkleization,
+  ./extras,
+  spec/[keystore, signatures],
   spec/datatypes/base
 
 func get_eth1data_stub*(deposit_count: uint64, current_epoch: Epoch): Eth1Data =

beacon_chain/networking/eth2_network.nim

@@ -36,7 +36,7 @@ import
     version, conf,
     ssz/ssz_serialization, beacon_clock],
   ../spec/datatypes/[phase0, altair],
-  ../spec/[digest, network, helpers, forks],
+  ../spec/[network, helpers, forks],
   ../validators/keystore_management,
   ./eth2_discovery, ./peer_pool, ./libp2p_json_serialization
 

beacon_chain/networking/network_metadata.nim

@@ -15,9 +15,9 @@ import
   chronicles,
   json_serialization,
   json_serialization/std/[options, sets, net], serialization/errors,
-  ../ssz/navigator,
+  ../ssz/[navigator, spec_types],
   eth/common/eth_types_json_serialization,
-  ../spec/[presets, datatypes, digest]
+  ../spec/datatypes/phase0
 
 # ATTENTION! This file will produce a large C file, because we are inlining
 # genesis states as C literals in the generated code (and blobs in the final
@@ -200,4 +200,4 @@ proc getRuntimeConfig*(
 
 proc extractGenesisValidatorRootFromSnapshop*(
     snapshot: string): Eth2Digest {.raises: [Defect, IOError, SszError].} =
-  sszMount(snapshot, BeaconState).genesis_validators_root[]
+  sszMount(snapshot, phase0.BeaconState).genesis_validators_root[]

beacon_chain/nimbus_beacon_node.nim

@@ -23,11 +23,9 @@ import
 
   # Local modules
   "."/[
-    beacon_chain_db,
-    beacon_node_common, beacon_node_status, beacon_node_types, conf,
-    extras, filepath, interop,
-    nimbus_binary_common, ssz/merkleization, statusbar,
-    beacon_clock, version],
+    beacon_clock, beacon_chain_db, beacon_node_common, beacon_node_status,
+    beacon_node_types, conf, filepath, interop, nimbus_binary_common, statusbar,
+    version],
   ./networking/[eth2_discovery, eth2_network, network_metadata],
   ./gossip_processing/[eth2_processor, block_processor, consensus_manager],
   ./validators/[
@@ -36,10 +34,9 @@ import
   ./sync/[sync_manager, sync_protocol, request_manager],
   ./rpc/[rest_api, rpc_api],
   ./spec/datatypes/[altair, phase0],
-  ./spec/[
-    digest, crypto, forks, beaconstate,
-    eth2_apis/rpc_beacon_client, helpers, network, presets,
-    weak_subjectivity, signatures],
+  ./spec/eth2_apis/rpc_beacon_client,
+  ./spec/[beaconstate, forks, helpers, network, weak_subjectivity, signatures],
+  ./ssz/merkleization,
   ./consensus_object_pools/[
     blockchain_dag, block_quarantine, block_clearance, block_pools_types,
     attestation_pool, exit_pool, spec_cache],

beacon_chain/nimbus_binary_common.nim

@@ -19,8 +19,10 @@ import
   stew/io2,
 
   # Local modules
-  ./spec/[crypto, helpers], ./spec/datatypes/base, beacon_clock, filepath,
-  ./beacon_node_status, ./networking/eth2_network
+  ./spec/[helpers],
+  ./spec/datatypes/base,
+  "."/[beacon_clock, filepath, beacon_node_status],
+  ./networking/eth2_network
 
 when defined(posix):
   import termios

beacon_chain/nimbus_signing_process.nim

@@ -9,7 +9,7 @@
 
 import
   # Standard library
-  os, strutils, tables,
+  std/[os, strutils, tables],
 
   # Local modules
   ./spec/[digest, crypto],

beacon_chain/rpc/eth_merge_web3.nim

@@ -1,8 +1,8 @@
 import
   strutils,
   json_serialization/std/[sets, net], serialization/errors,
-  ../spec/datatypes/base,
-  ../spec/[crypto, digest, eth2_apis/rpc_beacon_client],
+  ../spec/datatypes/[base, merge],
+  ../spec/eth2_apis/rpc_beacon_client,
   json_rpc/[client, jsonmarshal]
 
 from os import DirSep, AltSep

beacon_chain/rpc/rest_beacon_api.nim

@@ -12,7 +12,7 @@ import
   ../consensus_object_pools/[blockchain_dag, exit_pool],
   ../gossip_processing/gossip_validation,
   ../validators/validator_duties,
-  ../spec/[crypto, digest, forks, network],
+  ../spec/[forks, network],
   ../spec/datatypes/[phase0, altair],
   ../ssz/merkleization,
   ./rest_utils

beacon_chain/rpc/rest_config_api.nim

@@ -10,8 +10,7 @@ import
   chronicles,
   nimcrypto/utils as ncrutils,
   ../beacon_node_common, ../eth1/eth1_monitor,
-  ../spec/datatypes/base,
-  ../spec/[digest, forks, presets],
+  ../spec/forks,
   ./rest_utils
 
 logScope: topics = "rest_config"

beacon_chain/rpc/rest_debug_api.nim

@@ -3,7 +3,6 @@ import
   presto,
   chronicles,
   ../version, ../beacon_node_common,
-  ../spec/[datatypes, digest, presets],
   ./rest_utils
 
 logScope: topics = "rest_debug"

beacon_chain/rpc/rest_node_api.nim

@@ -8,7 +8,6 @@ import
   ../version, ../beacon_node_common, ../sync/sync_manager,
   ../networking/[eth2_network, peer_pool],
   ../spec/datatypes/base,
-  ../spec/[digest, presets],
   ../spec/eth2_apis/rpc_types,
   ./rest_utils
 

beacon_chain/rpc/rest_utils.nim

@@ -1,6 +1,6 @@
 import presto, presto/client as presto_client,
        nimcrypto/utils as ncrutils,
-       ../spec/[crypto, datatypes, digest, forks],
+       ../spec/[forks],
        ../spec/eth2_apis/[rest_types, eth2_rest_serialization],
        ../beacon_node_common,
        ../consensus_object_pools/[block_pools_types, blockchain_dag]