Corfu IPv6 (#3475)

Add support for IPv6 Addresses in addition to IPv4.
IPv6 addresses are prioritized over IPv4.

.ci/infrastructure-docker-build.sh

@@ -5,7 +5,7 @@ set -e
 PROFILE=$1
 BASE_IMAGE=$2
 if [ -z "$PROFILE" ]; then
-  echo "Please provide a profile name: 'docker' or 'compatibility'"
+  echo "Usage: './infrastructure-docker-build.sh <profile>', available profiles: docker, compatibility"
   exit 1
 fi
 

cmdlets/src/main/clojure/org/corfudb/shell.clj

@@ -1,5 +1,6 @@
 (ns org.corfudb.shell)
 
+(import org.corfudb.common.util.URLUtils)
 (import org.corfudb.runtime.CorfuRuntime)
 (import org.corfudb.runtime.clients.NettyClientRouter)
 (import org.docopt.Docopt)
@@ -90,8 +91,8 @@ The variable *r holds the last runtime obtrained, and *o holds the last router o
 
 
 ; Util functions to get a host or port from an endpoint string.
-(defn get-port [endpoint] (Integer/parseInt (get (.. endpoint (split ":")) 1)))
-(defn get-host [endpoint] (get (.. endpoint (split ":")) 0))
+(defn get-port [endpoint] (URLUtils/getPortFromEndpointURL (endpoint)))
+(defn get-host [endpoint] (URLUtils/getVersionFormattedHostAddress (endpoint)))
 
 ; Get a runtime or a router, and add it to the runtime/router corfuTable
 (defn add-client ([client] (.. *o (addClient client)))

common/src/main/java/org/corfudb/common/util/URLUtils.java

@@ -0,0 +1,166 @@
+package org.corfudb.common.util;
+
+import io.netty.channel.ChannelHandlerContext;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.conn.util.InetAddressUtils;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+
+/**
+ * Contains Utility methods to work with URIs or the Socket Addresses.
+ * Supports both IPv4 and IPv6 methods.
+ *
+ * Created by cgudisagar on 1/22/23.
+ */
+@Slf4j
+public final class URLUtils {
+
+    private static final String COLON_SEPERATOR = ":";
+
+    private URLUtils() {
+        // prevent instantiation of this class
+    }
+
+    /**
+     * Returns a version-formatted URL that contains the formatted host address along with the port.
+     *
+     * @param host host address that needs formatting
+     * @param port port of the endpoint
+     * @return a version-formatted endpoint
+     */
+    public static String getVersionFormattedEndpointURL(String host, Integer port) {
+        return getVersionFormattedEndpointURL(host, port.toString());
+    }
+
+    /**
+     * Returns a version-formatted URL that contains the formatted host address along with the port.
+     *
+     * @param host host address that needs formatting
+     * @param port port of the endpoint
+     * @return a version-formatted endpoint
+     */
+    public static String getVersionFormattedEndpointURL(String host, String port) {
+        return getVersionFormattedHostAddress(host) +
+                COLON_SEPERATOR +
+                port;
+    }
+
+    /**
+     * Returns a version-formatted URL that contains the formatted host address along with the port.
+     *
+     * @param address host address that needs formatting
+     * @return a version-formatted endpoint
+     */
+    public static String getVersionFormattedEndpointURL(String address) {
+        return getVersionFormattedHostAddress(address.substring(0, address.lastIndexOf(':'))) +
+                address.substring(address.lastIndexOf(COLON_SEPERATOR));
+    }
+
+    /**
+     * Returns a version-formatted URL that contains the formatted host address.
+     *
+     * @param host host address that needs formatting
+     * @return version-formatted address
+     */
+    public static String getVersionFormattedHostAddress(String host) {
+
+        // getByName(host) fails when host has scope/interface in it like ([....%eth0])
+        // remove the trailing names %eth0 or %en0 if present
+        String formattedHost = host.trim().split("%")[0];
+
+        try {
+            if (InetAddressUtils.isIPv6Address(InetAddress.getByName(formattedHost).getHostAddress())
+                && formattedHost.charAt(0)!='[' && formattedHost.charAt(formattedHost.length()-1)!=']') {
+                formattedHost = '[' + formattedHost + ']';
+            }
+        } catch (UnknownHostException e) {
+            log.warn("Unable to validate the host address: " + formattedHost, e);
+            return host;
+        }
+
+        return formattedHost;
+    }
+
+    /**
+     * Return the local socket address extracted from the netty Ctx
+     *
+     * @param ctx ChannelHandlerContext
+     * @return string in the form of IP:PORT
+     */
+    public static String getLocalEndpointFromCtx(ChannelHandlerContext ctx) {
+        try {
+            return getVersionFormattedHostAddress(((InetSocketAddress) ctx.channel().localAddress()).getAddress().getHostAddress())
+                    + COLON_SEPERATOR
+                    + ((InetSocketAddress) ctx.channel().localAddress()).getPort();
+        } catch (NullPointerException ex) {
+            return "unavailable";
+        }
+    }
+
+    /**
+     * Return the remote socket address extracted from the netty Ctx
+     *
+     * @param ctx ChannelHandlerContext
+     * @return string in the form of IP:PORT
+     */
+    public static String getRemoteEndpointFromCtx(ChannelHandlerContext ctx) {
+        try {
+            return getVersionFormattedHostAddress(
+                    ((InetSocketAddress) ctx.channel().remoteAddress()).getAddress().getHostAddress()
+            )
+                    + COLON_SEPERATOR
+                    + ((InetSocketAddress) ctx.channel().remoteAddress()).getPort();
+        } catch (NullPointerException ex) {
+            return "unavailable";
+        }
+    }
+
+    /**
+     * Extracts the host from the IPV4 or IPV6 endpoint URL
+     *
+     * @param address IPV4 or IPV6 endpoint URL
+     * @return extracted host address
+     */
+    public static String getHostFromEndpointURL(String address) {
+        return extractionHelper(true, address);
+    }
+
+    /**
+     * Extracts the port from the IPV4 or IPV6 endpoint URL
+     *
+     * @param address IPV4 or IPV6 endpoint URL
+     * @return extracted port
+     */
+    public static String getPortFromEndpointURL(String address) {
+        return extractionHelper(false, address);
+    }
+
+    /**
+     * A helper method to extract host and ports
+     * @param extractHost true to extract a host, false for port
+     * @param address address to extract form
+     * @return the extracted result
+     */
+    private static String extractionHelper(boolean extractHost, String address) {
+        int lastColonIndex = address.lastIndexOf(COLON_SEPERATOR);
+        // if ':' is present, return a substring
+        if (lastColonIndex != -1) {
+            if (extractHost) {
+                return address.substring(0, lastColonIndex);
+            } else {
+                return address.substring(lastColonIndex + 1);
+            }
+        } else {
+            log.warn("extractionHelper: Could not find colon in the address '{}'.", address);
+            return address;
+        }
+    }
+
+    public static enum NetworkInterfaceVersion {
+        IPV4,
+        IPV6
+    }
+
+}

common/src/test/java/org/corfudb/common/util/URLUtilsTest.java

@@ -0,0 +1,173 @@
+package org.corfudb.common.util;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.net.InetSocketAddress;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test class to test methods of {@link URLUtils} class.
+ * Created by cgudisagar on 2/3/23.
+ */
+public class URLUtilsTest {
+    private static final String IPV4_ADDRESS_1 = "127.0.0.1";
+    private static final String IPV4_ADDRESS_2 = "127.0.0.2";
+    private static final String IPV6_ADDRESS_1 = "[0:0:0:0:0:0:0:1]";
+    private static final String IPV6_ADDRESS_2 = "[0:0:0:0:0:0:0:2]";
+    private static final String IPV6_ADDRESS_1_SHORT = "[::1]";
+    private static final String IPV6_ADDRESS_1_MALFORMED = "::1";
+    private static final String IPV6_ADDRESS_2_SHORT = "[::2]";
+    private static final String IPV6_ADDRESS_2_MALFORMED = "::2";
+    private static final String LOCALHOST = "localhost";
+    private static final String PORT_STRING_0 = "9000";
+    private static final String PORT_STRING_1 = "9001";
+    private static final int PORT_INT_9000 = 9000;
+    private static final int PORT_INT_9001 = 9001;
+    private static final String LOCALHOST_ENDPOINT_URL = "localhost:9000";
+    private static final String IPV4_ENDPOINT_URL_1 = "127.0.0.1:9000";
+    private static final String IPV4_ENDPOINT_URL_2 = "127.0.0.2:9001";
+    private static final String IPV6_ENDPOINT_URL_1 = "[0:0:0:0:0:0:0:1]:9000";
+    private static final String IPV6_ENDPOINT_URL_2 = "[0:0:0:0:0:0:0:2]:9001";
+    private static final String IPV6_ENDPOINT_URL_1_SHORT = "[::1]:9000";
+    private static final String IPV6_ENDPOINT_URL_3_SHORT_MALFORMED = "::1:9000";
+    private static final String IPV6_ENDPOINT_URL_MALFORMED_1 = "0:0:0:0:0:0:0:1:9000";
+    private static final String IPV6_ENDPOINT_URL_MALFORMED_2 = "0:0:0:0:0:0:0:2:9001";
+
+    /**
+     * Utility method to get a mock ChannelHandlerContext object.
+     */
+    private static ChannelHandlerContext getMockChannelHandlerContext(
+            String localAddress, int localPort, String remoteAddress, int remotePort) {
+        ChannelHandlerContext ctx;
+        ctx = Mockito.mock(ChannelHandlerContext.class);
+        Channel ch = Mockito.mock(Channel.class);
+        when(ch.localAddress()).thenReturn(new InetSocketAddress(localAddress, localPort));
+        when(ch.remoteAddress()).thenReturn(new InetSocketAddress(remoteAddress, remotePort));
+        when(ctx.channel()).thenReturn(ch);
+        return ctx;
+    }
+
+    /**
+     * Test that version {@link URLUtils#getVersionFormattedEndpointURL} returns a version-formatted URL
+     * that contains the formatted host address along with the port.
+     */
+    @Test
+    void testGetVersionFormattedEndpointURL() {
+        assertThat(URLUtils.getVersionFormattedEndpointURL(LOCALHOST, PORT_INT_9000))
+                .isEqualTo(LOCALHOST_ENDPOINT_URL);
+        assertThat(URLUtils.getVersionFormattedEndpointURL(LOCALHOST, PORT_STRING_0))
+                .isEqualTo(LOCALHOST_ENDPOINT_URL);
+        assertThat(URLUtils.getVersionFormattedEndpointURL(IPV4_ENDPOINT_URL_1))
+                .isEqualTo(IPV4_ENDPOINT_URL_1);
+        assertThat(URLUtils.getVersionFormattedEndpointURL(IPV6_ENDPOINT_URL_1))
+                .isEqualTo(IPV6_ENDPOINT_URL_1);
+        assertThat(URLUtils.getVersionFormattedEndpointURL(IPV6_ENDPOINT_URL_MALFORMED_1))
+                .isEqualTo(IPV6_ENDPOINT_URL_1);
+        assertThat(URLUtils.getVersionFormattedEndpointURL(IPV6_ENDPOINT_URL_MALFORMED_2))
+                .isEqualTo(IPV6_ENDPOINT_URL_2);
+        assertThat(URLUtils.getVersionFormattedEndpointURL(IPV6_ENDPOINT_URL_3_SHORT_MALFORMED))
+                .isEqualTo(IPV6_ENDPOINT_URL_1_SHORT);
+    }
+
+    /**
+     * Test that version {@link URLUtils#getVersionFormattedHostAddress} returns a version-formatted URL
+     * that contains the formatted host address.
+     */
+    @Test
+    void testGetVersionFormattedHostAddress() {
+        // Should not alter IPv4 Addresses
+        assertThat(URLUtils.getVersionFormattedHostAddress(IPV4_ADDRESS_1))
+                .isEqualTo(IPV4_ADDRESS_1);
+
+        // Should not alter IPv4 Addresses
+        assertThat(URLUtils.getVersionFormattedHostAddress(IPV4_ADDRESS_2))
+                .isEqualTo(IPV4_ADDRESS_2);
+
+        // Should not alter version-formatted IPv6 Address
+        assertThat(URLUtils.getVersionFormattedHostAddress(IPV6_ADDRESS_1_SHORT))
+                .isEqualTo(IPV6_ADDRESS_1_SHORT);
+
+        // Should not alter version-formatted IPv6 Address
+        assertThat(URLUtils.getVersionFormattedHostAddress(IPV6_ADDRESS_2_SHORT))
+                .isEqualTo(IPV6_ADDRESS_2_SHORT);
+
+        // Should not alter version-formatted IPv6 Address
+        assertThat(URLUtils.getVersionFormattedHostAddress(LOCALHOST))
+                .isEqualTo(LOCALHOST);
+
+        // Should return version-formatted IPv6 Address with '[' and ']'
+        assertThat(URLUtils.getVersionFormattedHostAddress(IPV6_ADDRESS_1_MALFORMED))
+                .isEqualTo(IPV6_ADDRESS_1_SHORT);
+
+        // Should return version-formatted IPv6 Address with '[' and ']'
+        assertThat(URLUtils.getVersionFormattedHostAddress(IPV6_ADDRESS_2_MALFORMED))
+                .isEqualTo(IPV6_ADDRESS_2_SHORT);
+    }
+
+    /**
+     * Test that {@link URLUtils#getLocalEndpointFromCtx} returns the endpoint containing
+     * the local socket address extracted from the netty Ctx
+     */
+    @Test
+    void testGetLocalEndpointFromCtx() {
+        ChannelHandlerContext ctx =
+                getMockChannelHandlerContext(IPV4_ADDRESS_1, PORT_INT_9000, IPV4_ADDRESS_1, PORT_INT_9000);
+        assertThat(URLUtils.getLocalEndpointFromCtx(ctx)).isEqualTo(IPV4_ENDPOINT_URL_1);
+
+        // IPv6
+        ctx = getMockChannelHandlerContext(IPV6_ADDRESS_1_SHORT, PORT_INT_9000, IPV6_ADDRESS_2_SHORT, PORT_INT_9001);
+        assertThat(URLUtils.getLocalEndpointFromCtx(ctx)).isEqualTo(IPV6_ENDPOINT_URL_1);
+    }
+
+    /**
+     * Test that {@link URLUtils#getRemoteEndpointFromCtx} returns the endpoint containing
+     * the remote socket address extracted from the netty Ctx
+     */
+    @Test
+    void getRemoteEndpointFromCtx() {
+        ChannelHandlerContext ctx =
+                getMockChannelHandlerContext(IPV4_ADDRESS_1, PORT_INT_9000, IPV4_ADDRESS_2, PORT_INT_9001);
+        assertThat(URLUtils.getRemoteEndpointFromCtx(ctx)).isEqualTo(IPV4_ENDPOINT_URL_2);
+
+        // IPv6
+        ctx = getMockChannelHandlerContext(IPV6_ADDRESS_1_SHORT, PORT_INT_9000, IPV6_ADDRESS_2_SHORT, PORT_INT_9001);
+        assertThat(URLUtils.getRemoteEndpointFromCtx(ctx)).isEqualTo(IPV6_ENDPOINT_URL_2);
+    }
+
+    /**
+     * Test that {@link URLUtils#getPortFromEndpointURL} extracts the port from
+     * the IPV4 or IPV6 endpoint URL
+     */
+    @Test
+    void testGetPortFromEndpointURL() {
+        assertThat(URLUtils.getPortFromEndpointURL(IPV4_ENDPOINT_URL_1)).isEqualTo(PORT_STRING_0);
+        assertThat(URLUtils.getPortFromEndpointURL(IPV4_ENDPOINT_URL_2)).isEqualTo(PORT_STRING_1);
+        assertThat(URLUtils.getPortFromEndpointURL(IPV6_ENDPOINT_URL_1)).isEqualTo(PORT_STRING_0);
+        assertThat(URLUtils.getPortFromEndpointURL(IPV6_ENDPOINT_URL_2)).isEqualTo(PORT_STRING_1);
+        assertThat(URLUtils.getPortFromEndpointURL(IPV6_ENDPOINT_URL_1_SHORT)).isEqualTo(PORT_STRING_0);
+        // no colon
+        assertThat(URLUtils.getPortFromEndpointURL(IPV4_ADDRESS_1)).isEqualTo(IPV4_ADDRESS_1);
+        assertThat(URLUtils.getPortFromEndpointURL(PORT_STRING_0)).isEqualTo(PORT_STRING_0);
+    }
+
+    /**
+     * Test that {@link URLUtils#getPortFromEndpointURL} extracts the port from
+     * the IPV4 or IPV6 endpoint URL
+     */
+    @Test
+    void testGetHostFromEndpointURL() {
+        assertThat(URLUtils.getHostFromEndpointURL(IPV4_ENDPOINT_URL_1)).isEqualTo(IPV4_ADDRESS_1);
+        assertThat(URLUtils.getHostFromEndpointURL(IPV4_ENDPOINT_URL_2)).isEqualTo(IPV4_ADDRESS_2);
+        assertThat(URLUtils.getHostFromEndpointURL(IPV6_ENDPOINT_URL_1)).isEqualTo(IPV6_ADDRESS_1);
+        assertThat(URLUtils.getHostFromEndpointURL(IPV6_ENDPOINT_URL_2)).isEqualTo(IPV6_ADDRESS_2);
+        assertThat(URLUtils.getHostFromEndpointURL(IPV6_ENDPOINT_URL_1_SHORT)).isEqualTo(IPV6_ADDRESS_1_SHORT);
+        // no colon
+        assertThat(URLUtils.getPortFromEndpointURL(IPV4_ADDRESS_1)).isEqualTo(IPV4_ADDRESS_1);
+        assertThat(URLUtils.getPortFromEndpointURL(PORT_STRING_0)).isEqualTo(PORT_STRING_0);
+    }
+}

corfu_scripts/corfu_reset.clj

@@ -22,7 +22,7 @@ Options:
 ; Parse the incoming docopt options.
 (def localcmd (.. (new Docopt usage) (parse *args)))
 
-(defn print-reset [endpoint] (do (println (str "Reset " endpoint ":"))
+(defn print-reset [endpoint] (do (println (str "Reset " endpoint ": "))
                                  (if (.. (.. (get-base-client (get-router server localcmd) 0 (.UUID (.fromString "00000000-0000-0000-0000-000000000000"))) (reset)) (get))
                                      (println "ACK")
                                      (println "NACK")