From be962a04717dcc88266a79402286290d1e921cc2 Mon Sep 17 00:00:00 2001 From: Edward Wertz <123979964+edward-swirldslabs@users.noreply.github.com> Date: Wed, 20 Dec 2023 10:09:54 -0600 Subject: [PATCH] feat: design roster interfaces (#10428) Signed-off-by: Edward Wertz --- .../roster/roster-datastructures-and-apis.md | 31 ++++ .../com/swirlds/platform/roster/Roster.java | 69 ++++++++ .../swirlds/platform/roster/RosterEntry.java | 73 ++++++++ .../roster/legacy/AddressBookRoster.java | 162 ++++++++++++++++++ .../roster/legacy/AddressRosterEntry.java | 162 ++++++++++++++++++ .../roster/legacy/AddressBookRosterTests.java | 84 +++++++++ .../system/address/AddressBookTests.java | 5 +- 7 files changed, 582 insertions(+), 4 deletions(-) create mode 100644 platform-sdk/docs/core/dynamic-address-book/roster/roster-datastructures-and-apis.md create mode 100644 platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/Roster.java create mode 100644 platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/RosterEntry.java create mode 100644 platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/legacy/AddressBookRoster.java create mode 100644 platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/legacy/AddressRosterEntry.java create mode 100644 platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/roster/legacy/AddressBookRosterTests.java diff --git a/platform-sdk/docs/core/dynamic-address-book/roster/roster-datastructures-and-apis.md b/platform-sdk/docs/core/dynamic-address-book/roster/roster-datastructures-and-apis.md new file mode 100644 index 000000000000..d3eceaaafb7a --- /dev/null +++ b/platform-sdk/docs/core/dynamic-address-book/roster/roster-datastructures-and-apis.md @@ -0,0 +1,31 @@ +# Roster APIs + +The following roster api is reduced from the address book to just the fields that are needed by the platform to establish mutual TLS connections, gossip, validate events and state, come to consensus, and detect an ISS. + +The data for each node is contained in the node's `RosterEntry`. + +## Roster Interfaces + +### RosterEntry + +```java +public interface RosterEntry extends SelfSerializable { + NodeId getNodeId(); + long getWeight(); + String getHostname(); + int getPort(); + PublicKey getSigningPublicKey(); + X509Certificate getSigningCertificate(); + boolean isZeroWeight(); +} +``` +### Roster + +```java +public interface Roster extends Iterable, SelfSerializable{ + int size(); + boolean contains(NodeId nodeId); + RosterEntry getEntry(NodeId nodeId); + long getTotalWeight(); +} +``` \ No newline at end of file diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/Roster.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/Roster.java new file mode 100644 index 000000000000..5e14a90e2e9a --- /dev/null +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/Roster.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.platform.roster; + +import com.swirlds.common.io.SelfSerializable; +import com.swirlds.common.platform.NodeId; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.Collection; + +/** + * A roster is the set of nodes that are creating events and contributing to consensus. The data in a Roster object is + * immutable must not change over time. + */ +public interface Roster extends Iterable, SelfSerializable { + + /** + * @return a collection of all unique nodeIds in the roster. + */ + @NonNull + Collection getNodeIds(); + + /** + * @param nodeId the nodeId of the {@link RosterEntry} to get + * @return the RosterEntry with the given nodeId + * @throws java.util.NoSuchElementException if the nodeId is not in the roster + */ + @NonNull + RosterEntry getEntry(@NonNull NodeId nodeId); + + /** + * @param nodeId the nodeId to check for membership in the roster + * @return true if there is a rosterEntry with the given nodeId, false otherwise + */ + default boolean contains(@NonNull NodeId nodeId) { + return getNodeIds().contains(nodeId); + } + + /** + * @return the total number of nodes in the roster + */ + default int getSize() { + return getNodeIds().size(); + } + + /** + * @return the total weight of all nodes in the roster + */ + default long getTotalWeight() { + long totalWeight = 0; + for (final RosterEntry entry : this) { + totalWeight += entry.getWeight(); + } + return totalWeight; + } +} diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/RosterEntry.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/RosterEntry.java new file mode 100644 index 000000000000..0f15497f6d71 --- /dev/null +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/RosterEntry.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.platform.roster; + +import com.swirlds.common.io.SelfSerializable; +import com.swirlds.common.platform.NodeId; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.security.PublicKey; +import java.security.cert.X509Certificate; + +/** + * A RosterEntry is a single node in the roster. It contains the node's ID, weight, network address, and public signing + * key in the form of an X509Certificate. The data in a RosterEntry object is immutable and must not change over time. + */ +public interface RosterEntry extends SelfSerializable { + + /** + * @return the ID of the node + */ + @NonNull + NodeId getNodeId(); + + /** + * @return the non-negative consensus weight of the node + */ + long getWeight(); + + /** + * @return the hostname portion of a node's gossip endpoint. + */ + @NonNull + String getHostname(); + + /** + * @return the port portion of a node's gossip endpoint. + */ + int getPort(); + + /** + * @return the X509Certificate containing the public signing key of the node + */ + @NonNull + X509Certificate getSigningCertificate(); + + /** + * @return the public signing key of the node + */ + @NonNull + default PublicKey getSigningPublicKey() { + return getSigningCertificate().getPublicKey(); + } + + /** + * @return true if the weight is zero, false otherwise + */ + default boolean isZeroWeight() { + return getWeight() == 0; + } +} diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/legacy/AddressBookRoster.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/legacy/AddressBookRoster.java new file mode 100644 index 000000000000..d7833dd708f7 --- /dev/null +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/legacy/AddressBookRoster.java @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.platform.roster.legacy; + +import com.swirlds.base.utility.ToStringBuilder; +import com.swirlds.common.io.streams.SerializableDataInputStream; +import com.swirlds.common.io.streams.SerializableDataOutputStream; +import com.swirlds.common.platform.NodeId; +import com.swirlds.platform.crypto.KeysAndCerts; +import com.swirlds.platform.roster.Roster; +import com.swirlds.platform.roster.RosterEntry; +import com.swirlds.platform.system.address.Address; +import com.swirlds.platform.system.address.AddressBook; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; + +/** + * A {@link Roster} implementation that uses an {@link AddressBook} as its backing data structure. + */ +public class AddressBookRoster implements Roster { + private static final long CLASS_ID = 0x7104f97d4e298619L; + + private static final class ClassVersion { + public static final int ORIGINAL = 1; + } + + private final Map entries = new HashMap<>(); + private List nodeOrder; + + /** + * Constructs a new {@link AddressBookRoster} from the given {@link AddressBook} and {@link KeysAndCerts} map. + * + * @param addressBook the address book + * @param keysAndCertsMap the keys and certs map + */ + public AddressBookRoster( + @NonNull final AddressBook addressBook, @NonNull final Map keysAndCertsMap) { + Objects.requireNonNull(addressBook); + Objects.requireNonNull(keysAndCertsMap); + + for (final Address address : addressBook) { + entries.put(address.getNodeId(), new AddressRosterEntry(address, keysAndCertsMap.get(address.getNodeId()))); + } + + nodeOrder = entries.keySet().stream().sorted().toList(); + } + + /** + * Empty constructor for deserialization. + */ + public AddressBookRoster() { + nodeOrder = new ArrayList<>(); + } + + @Override + public long getClassId() { + return CLASS_ID; + } + + @Override + public int getVersion() { + return ClassVersion.ORIGINAL; + } + + @Override + public void serialize(@NonNull final SerializableDataOutputStream out) throws IOException { + out.writeInt(entries.size()); + for (final RosterEntry entry : this) { + out.writeSerializable(entry, true); + } + } + + @Override + public void deserialize(@NonNull final SerializableDataInputStream in, final int version) throws IOException { + final int size = in.readInt(); + for (int i = 0; i < size; i++) { + final RosterEntry entry = in.readSerializable(); + entries.put(entry.getNodeId(), entry); + } + nodeOrder = entries.keySet().stream().sorted().toList(); + } + + @Override + @NonNull + public Collection getNodeIds() { + return nodeOrder; + } + + @Override + @NonNull + public RosterEntry getEntry(@NonNull final NodeId nodeId) { + Objects.requireNonNull(nodeId); + final RosterEntry entry = entries.get(nodeId); + if (entry == null) { + throw new NoSuchElementException("No entry found for nodeId " + nodeId); + } + return entry; + } + + @Override + @NonNull + public Iterator iterator() { + return new Iterator<>() { + private int index = 0; + + @Override + public boolean hasNext() { + return index < nodeOrder.size(); + } + + @Override + public RosterEntry next() { + return entries.get(nodeOrder.get(index++)); + } + }; + } + + @Override + public boolean equals(@Nullable final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AddressBookRoster that = (AddressBookRoster) o; + return Objects.equals(entries, that.entries); + } + + @Override + public int hashCode() { + return Objects.hash(entries); + } + + @Override + public String toString() { + return new ToStringBuilder(this).append("entries", entries).toString(); + } +} diff --git a/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/legacy/AddressRosterEntry.java b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/legacy/AddressRosterEntry.java new file mode 100644 index 000000000000..24eb86a7cd45 --- /dev/null +++ b/platform-sdk/swirlds-platform-core/src/main/java/com/swirlds/platform/roster/legacy/AddressRosterEntry.java @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.platform.roster.legacy; + +import com.swirlds.base.utility.ToStringBuilder; +import com.swirlds.common.io.streams.SerializableDataInputStream; +import com.swirlds.common.io.streams.SerializableDataOutputStream; +import com.swirlds.common.platform.NodeId; +import com.swirlds.platform.crypto.KeysAndCerts; +import com.swirlds.platform.roster.RosterEntry; +import com.swirlds.platform.system.address.Address; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.security.cert.CertificateEncodingException; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.Objects; + +/** + * An {@link Address} wrapper that implements the {@link RosterEntry} interface. + */ +public class AddressRosterEntry implements RosterEntry { + + private static final long CLASS_ID = 0x4e700e352be188aaL; + + private static final class ClassVersion { + public static final int ORIGINAL = 1; + } + + private static final int ENCODED_CERT_MAX_SIZE = 8192; + + private Address address; + private X509Certificate sigCert; + + /** + * Constructs a new {@link AddressRosterEntry} from the given {@link Address} and {@link KeysAndCerts}. + * + * @param address the address + * @param keysAndCerts the keys and certs containing the signing certificate + */ + public AddressRosterEntry(@NonNull final Address address, @NonNull final KeysAndCerts keysAndCerts) { + Objects.requireNonNull(address); + Objects.requireNonNull(keysAndCerts); + + this.address = address; + this.sigCert = keysAndCerts.sigCert(); + } + + /** + * Empty constructor for deserialization. + */ + public AddressRosterEntry() {} + + /** + * {@inheritDoc} + */ + @Override + public long getClassId() { + return CLASS_ID; + } + + /** + * {@inheritDoc} + */ + @Override + public int getVersion() { + return ClassVersion.ORIGINAL; + } + + @Override + public void serialize(@NonNull final SerializableDataOutputStream out) throws IOException { + out.writeSerializable(address, false); + try { + out.writeByteArray(sigCert.getEncoded()); + } catch (final CertificateEncodingException e) { + throw new IOException("Could not encode certificate", e); + } + } + + @Override + public void deserialize(@NonNull final SerializableDataInputStream in, final int version) throws IOException { + address = in.readSerializable(false, Address::new); + final byte[] encodedCert = in.readByteArray(ENCODED_CERT_MAX_SIZE); + try { + sigCert = (X509Certificate) + CertificateFactory.getInstance("X.509").generateCertificate(new ByteArrayInputStream(encodedCert)); + } catch (final CertificateException e) { + throw new IOException("Could not decode certificate", e); + } + } + + @Override + @NonNull + public NodeId getNodeId() { + return address.getNodeId(); + } + + @Override + public long getWeight() { + return address.getWeight(); + } + + @NonNull + @Override + public String getHostname() { + return Objects.requireNonNullElse(address.getHostnameExternal(), ""); + } + + @Override + public int getPort() { + return address.getPortExternal(); + } + + @NonNull + @Override + public X509Certificate getSigningCertificate() { + return sigCert; + } + + @Override + public boolean equals(@Nullable final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AddressRosterEntry that = (AddressRosterEntry) o; + return Objects.equals(address, that.address) && Objects.equals(sigCert, that.sigCert); + } + + @Override + public int hashCode() { + return Objects.hash(address, sigCert); + } + + @Override + public String toString() { + + return new ToStringBuilder(this) + .append("address", address) + .append("sigCert", sigCert) + .toString(); + } +} diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/roster/legacy/AddressBookRosterTests.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/roster/legacy/AddressBookRosterTests.java new file mode 100644 index 000000000000..26a06b406d81 --- /dev/null +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/roster/legacy/AddressBookRosterTests.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2023 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.swirlds.platform.roster.legacy; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import com.swirlds.common.constructable.ConstructableRegistry; +import com.swirlds.common.constructable.ConstructableRegistryException; +import com.swirlds.common.io.streams.SerializableDataInputStream; +import com.swirlds.common.io.streams.SerializableDataOutputStream; +import com.swirlds.common.platform.NodeId; +import com.swirlds.platform.crypto.KeysAndCerts; +import com.swirlds.platform.roster.Roster; +import com.swirlds.platform.roster.RosterEntry; +import com.swirlds.platform.system.address.Address; +import com.swirlds.platform.system.address.AddressBook; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +public class AddressBookRosterTests { + + @DisplayName("Serialize and deserialize AddressBook derived Roster") + @ParameterizedTest + @MethodSource({"com.swirlds.platform.crypto.CryptoArgsProvider#basicTestArgs"}) + void serializeDeserializeTest( + @NonNull final AddressBook addressBook, @NonNull final Map keysAndCerts) + throws IOException, ConstructableRegistryException { + ConstructableRegistry.getInstance().registerConstructables("com.swirlds"); + final Roster roster = new AddressBookRoster(addressBook, keysAndCerts); + + final ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + final SerializableDataOutputStream out = new SerializableDataOutputStream(byteOut); + + out.writeSerializable(roster, true); + + final SerializableDataInputStream in = + new SerializableDataInputStream(new ByteArrayInputStream(byteOut.toByteArray())); + + Roster roster2 = in.readSerializable(); + assertEquals(roster, roster2); + } + + @DisplayName("Roster derived from AddressBook") + @ParameterizedTest + @MethodSource({"com.swirlds.platform.crypto.CryptoArgsProvider#basicTestArgs"}) + void addressBookRosterTest( + @NonNull final AddressBook addressBook, @NonNull final Map keysAndCerts) { + final Roster roster = new AddressBookRoster(addressBook, keysAndCerts); + final Iterator entries = roster.iterator(); + for (int i = 0; i < addressBook.getSize(); i++) { + final NodeId nodeId = addressBook.getNodeId(i); + final Address address = addressBook.getAddress(nodeId); + final RosterEntry rosterEntry = entries.next(); + assertEquals(address.getHostnameExternal(), rosterEntry.getHostname()); + assertEquals(address.getPortExternal(), rosterEntry.getPort()); + assertEquals(address.getNodeId(), rosterEntry.getNodeId()); + assertEquals(address.getWeight(), rosterEntry.getWeight()); + assertEquals(address.getSigPublicKey(), rosterEntry.getSigningPublicKey()); + } + assertFalse(entries.hasNext()); + } +} diff --git a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/system/address/AddressBookTests.java b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/system/address/AddressBookTests.java index 5a6510d78b95..db64b8ec7845 100644 --- a/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/system/address/AddressBookTests.java +++ b/platform-sdk/swirlds-platform-core/src/test/java/com/swirlds/platform/system/address/AddressBookTests.java @@ -18,7 +18,6 @@ import static com.swirlds.common.test.fixtures.RandomUtils.getRandomPrintSeed; import static com.swirlds.platform.system.address.AddressBookUtils.parseAddressBookText; -import static com.swirlds.test.framework.TestQualifierTags.TIME_CONSUMING; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -48,7 +47,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @DisplayName("AddressBook Tests") @@ -329,9 +327,8 @@ void atoStringSanityTest() { @Test @DisplayName("Serialization Test") - @Tag(TIME_CONSUMING) void serializationTest() throws IOException, ConstructableRegistryException { - ConstructableRegistry.getInstance().registerConstructables("com.swirlds.common.system"); + ConstructableRegistry.getInstance().registerConstructables("com.swirlds"); final AddressBook original = new RandomAddressBookGenerator(getRandomPrintSeed()) .setSize(100)