Merge pull request #113 from yahoo/omid-38

[OMID-38] Align and improve client/server configurations

common/src/main/java/org/apache/omid/NetworkUtils.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.omid;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.util.Enumeration;
+
+public class NetworkUtils {
+
+    private static final Logger LOG = LoggerFactory.getLogger(NetworkUtils.class);
+
+    private static final String LINUX_TSO_NET_IFACE_PREFIX = "eth";
+    private static final String MAC_TSO_NET_IFACE_PREFIX = "en";
+
+    public static String getDefaultNetworkInterface() {
+
+        try {
+            Enumeration<NetworkInterface> networkInterfaces = NetworkInterface.getNetworkInterfaces();
+            while (networkInterfaces.hasMoreElements()) {
+                String name = networkInterfaces.nextElement().getDisplayName();
+                LOG.info("Iterating over network interfaces, found '{}'", name);
+                if (name.startsWith(MAC_TSO_NET_IFACE_PREFIX) || name.startsWith(LINUX_TSO_NET_IFACE_PREFIX)) {
+                    return name;
+                }
+            }
+        } catch (SocketException ignored) {
+            throw new RuntimeException("Failed to find any network interfaces", ignored);
+        }
+        throw new IllegalArgumentException(String.format("No network '%s*'/'%s*' interfaces found",
+                                                         MAC_TSO_NET_IFACE_PREFIX, LINUX_TSO_NET_IFACE_PREFIX));
+
+    }
+
+}

hbase-client/src/test/java/org/apache/omid/transaction/OmidTestBase.java

@@ -82,7 +82,7 @@ public void beforeGroups(ITestContext context) throws Exception {
         // TSO Setup
         TSOServerConfig tsoConfig = new TSOServerConfig();
         tsoConfig.setPort(1234);
-        tsoConfig.setMaxItems(1000);
+        tsoConfig.setConflictMapSize(1000);
         Injector injector = Guice.createInjector(new TSOMockModule(tsoConfig));
         LOG.info("Starting TSO");
         TSOServer tso = injector.getInstance(TSOServer.class);

hbase-client/src/test/java/org/apache/omid/transaction/TestEndToEndScenariosWithHA.java

@@ -115,7 +115,7 @@ public void nodeChanged() throws Exception {
         // Configure TSO 1
         TSOServerConfig config1 = new TSOServerConfig();
         config1.setPort(TSO1_PORT);
-        config1.setMaxItems(1000);
+        config1.setConflictMapSize(1000);
         config1.setLeaseModule(new TestHALeaseManagementModule(TEST_LEASE_PERIOD_MS, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkConnection, NAMESPACE));
         Injector injector1 = Guice.createInjector(new TestTSOModule(hbaseConf, config1));
         LOG.info("===================== Starting TSO 1 =====================");
@@ -128,7 +128,7 @@ public void nodeChanged() throws Exception {
         // Configure TSO 2
         TSOServerConfig config2 = new TSOServerConfig();
         config2.setPort(TSO2_PORT);
-        config2.setMaxItems(1000);
+        config2.setConflictMapSize(1000);
         config2.setLeaseModule(new TestHALeaseManagementModule(TEST_LEASE_PERIOD_MS, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkConnection, NAMESPACE));
         Injector injector2 = Guice.createInjector(new TestTSOModule(hbaseConf, config2));
         LOG.info("===================== Starting TSO 2 =====================");

hbase-coprocessor/src/test/java/org/apache/omid/transaction/TestCompaction.java

@@ -111,7 +111,7 @@ public class TestCompaction {
     public void setupTestCompation() throws Exception {
         TSOServerConfig tsoConfig = new TSOServerConfig();
         tsoConfig.setPort(1234);
-        tsoConfig.setMaxItems(1);
+        tsoConfig.setConflictMapSize(1);
         injector = Guice.createInjector(new TSOForHBaseCompactorTestModule(tsoConfig));
         hbaseConf = injector.getInstance(Configuration.class);
         HBaseCommitTableConfig hBaseCommitTableConfig = injector.getInstance(HBaseCommitTableConfig.class);

tso-server/bin/omid.sh

@@ -17,19 +17,21 @@
 # limitations under the License.
 #
 
-
 SCRIPTDIR=`dirname $0`
 cd $SCRIPTDIR;
-CLASSPATH=../conf:$(HBASE_CONF_DIR):$(HADOOP_CONF_DIR)
 
-. ../conf/omid-env.sh
+# Load Omid environment variables
+source omid-env.sh
+
+# Configure classpath...
+CLASSPATH=../conf:$(HBASE_CONF_DIR):$(HADOOP_CONF_DIR)
 
-# for source release
+# ...for source release and...
 for j in ../target/omid-tso*.jar; do
     CLASSPATH=$CLASSPATH:$j
 done
 
-# for binary release
+# and for binary release
 for j in ../omid-tso*.jar; do
     CLASSPATH=$CLASSPATH:$j
 done
@@ -38,7 +40,7 @@ for j in ../lib/*.jar; do
 done
 
 tso() {
-    exec java $JVM_FLAGS -cp $CLASSPATH org.apache.omid.tso.TSOServer @../conf/omid.conf $@
+    exec java $JVM_FLAGS -cp $CLASSPATH org.apache.omid.tso.TSOServer $@
 }
 
 tsoRelauncher() {

tso-server/conf/omid-server-configuration.yml

@@ -0,0 +1,8 @@
+# =====================================================================================================================
+#
+# Omid TSO Server Configuration
+# ---------------------------------------------------------------------------------------------------------------------
+#
+# Tune here the default values for TSO server config parameters found in 'default-omid-server-configuration.yml' file
+#
+# =====================================================================================================================

tso-server/src/main/java/org/apache/omid/tso/RequestProcessorImpl.java

@@ -89,7 +89,7 @@ class RequestProcessorImpl implements EventHandler<RequestProcessorImpl.RequestE
         this.metrics = metrics;
         this.persistProc = persistProc;
         this.timestampOracle = timestampOracle;
-        this.hashmap = new CommitHashMap(config.getMaxItems());
+        this.hashmap = new CommitHashMap(config.getConflictMapSize());
 
         LOG.info("RequestProcessor initialized");
 

tso-server/src/main/java/org/apache/omid/tso/TSOServerConfig.java

@@ -19,16 +19,13 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.inject.Module;
+import org.apache.omid.NetworkUtils;
 import org.apache.omid.YAMLUtils;
 import org.apache.omid.metrics.MetricsRegistry;
 import org.apache.omid.tools.hbase.SecureHBaseConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.net.NetworkInterface;
-import java.net.SocketException;
-import java.util.Enumeration;
-
 /**
  * Reads the configuration parameters of a TSO server instance from CONFIG_FILE_NAME.
  * If file CONFIG_FILE_NAME is missing defaults to DEFAULT_CONFIG_FILE_NAME
@@ -38,11 +35,8 @@ public class TSOServerConfig extends SecureHBaseConfig {
 
     private static final Logger LOG = LoggerFactory.getLogger(TSOServerConfig.class);
 
-    private static final String DEFAULT_CONFIG_FILE_NAME = "default-omid.yml";
-    private static final String CONFIG_FILE_NAME = "omid.yml";
-
-    private static final String LINUX_TSO_NET_IFACE_PREFIX = "eth";
-    private static final String MAC_TSO_NET_IFACE_PREFIX = "en";
+    private static final String CONFIG_FILE_NAME = "omid-server-configuration.yml";
+    private static final String DEFAULT_CONFIG_FILE_NAME = "default-omid-server-configuration.yml";
 
     // ----------------------------------------------------------------------------------------------------------------
     // Instantiation
@@ -70,15 +64,15 @@ public TSOServerConfig() {
 
     private MetricsRegistry metrics;
 
-    private int maxItems;
+    private int conflictMapSize;
 
     private int numConcurrentCTWriters;
 
     private int batchSizePerCTWriter;
 
     private int batchPersistTimeoutInMs;
 
-    private String networkIfaceName = getDefaultNetworkInterface();
+    private String networkIfaceName = NetworkUtils.getDefaultNetworkInterface();
 
     public int getPort() {
         return port;
@@ -88,12 +82,12 @@ public void setPort(int port) {
         this.port = port;
     }
 
-    public int getMaxItems() {
-        return maxItems;
+    public int getConflictMapSize() {
+        return conflictMapSize;
     }
 
-    public void setMaxItems(int maxItems) {
-        this.maxItems = maxItems;
+    public void setConflictMapSize(int conflictMapSize) {
+        this.conflictMapSize = conflictMapSize;
     }
 
     public int getNumConcurrentCTWriters() {
@@ -160,27 +154,4 @@ public void setMetrics(MetricsRegistry metrics) {
         this.metrics = metrics;
     }
 
-    // ----------------------------------------------------------------------------------------------------------------
-    // Helper methods
-    // ----------------------------------------------------------------------------------------------------------------
-
-    private String getDefaultNetworkInterface() {
-
-        try {
-            Enumeration<NetworkInterface> networkInterfaces = NetworkInterface.getNetworkInterfaces();
-            while (networkInterfaces.hasMoreElements()) {
-                String name = networkInterfaces.nextElement().getDisplayName();
-                LOG.info("Iterating over network interfaces, found '{}'", name);
-                if (name.startsWith(MAC_TSO_NET_IFACE_PREFIX) || name.startsWith(LINUX_TSO_NET_IFACE_PREFIX)) {
-                    return name;
-                }
-            }
-        } catch (SocketException ignored) {
-            throw new RuntimeException("Failed to find any network interfaces", ignored);
-        }
-        throw new IllegalArgumentException(String.format("No network '%s*'/'%s*' interfaces found",
-                                                         MAC_TSO_NET_IFACE_PREFIX, LINUX_TSO_NET_IFACE_PREFIX));
-
-    }
-
 }

tso-server/src/main/resources/default-omid.yml → tso-server/src/main/resources/default-omid-server-configuration.yml

@@ -1,5 +1,5 @@
 # =====================================================================================================================
-# Omid TSO Server Configuration
+# Omid TSO Server Configuration (Default parameters)
 # =====================================================================================================================
 
 # ---------------------------------------------------------------------------------------------------------------------
@@ -14,8 +14,7 @@
 # Port reserved by the Status Oracle
 port: 54758
 # The number of elements reserved in the conflict map to perform conflict resolution
-# TODO Rename to conflictMapSize
-maxItems: 100000000
+conflictMapSize: 100000000
 # The number of Commit Table writers that persist data concurrently to the datastore. It has to be at least 2.
 numConcurrentCTWriters: 2
 # The size of the batch of operations that each Commit Table writes has. The maximum number of operations that can be

tso-server/src/test/java/org/apache/omid/tso/TestRequestProcessor.java

@@ -32,7 +32,6 @@
 import java.util.List;
 
 import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.doReturn;
@@ -74,7 +73,7 @@ public void beforeMethod() throws Exception {
         doReturn(f).when(persist).persistLowWatermark(any(Long.class));
 
         TSOServerConfig config = new TSOServerConfig();
-        config.setMaxItems(CONFLICT_MAP_SIZE);
+        config.setConflictMapSize(CONFLICT_MAP_SIZE);
 
         requestProc = new RequestProcessorImpl(metrics, timestampOracle, persist, new MockPanicker(), config);
 

tso-server/src/test/java/org/apache/omid/tso/client/TestIntegrationOfTSOClientServerBasicFunctionality.java

@@ -66,7 +66,7 @@ public void setup() throws Exception {
         tsoServerPortForTest = TestUtils.getFreeLocalPort();
 
         TSOServerConfig tsoConfig = new TSOServerConfig();
-        tsoConfig.setMaxItems(1000);
+        tsoConfig.setConflictMapSize(1000);
         tsoConfig.setPort(tsoServerPortForTest);
         Module tsoServerMockModule = new TSOMockModule(tsoConfig);
         Injector injector = Guice.createInjector(tsoServerMockModule);

tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientConnectionToTSO.java

@@ -117,7 +117,7 @@ public void testSuccessfulConnectionToTSOWithHostAndPort() throws Exception {
 
         // Launch a TSO WITHOUT publishing the address in HA...
         TSOServerConfig tsoConfig = new TSOServerConfig();
-        tsoConfig.setMaxItems(1000);
+        tsoConfig.setConflictMapSize(1000);
         tsoConfig.setPort(tsoPortForTest);
         tsoConfig.setLeaseModule(new VoidLeaseManagementModule());
         injector = Guice.createInjector(new TSOMockModule(tsoConfig));
@@ -153,7 +153,7 @@ public void testSuccessfulConnectionToTSOThroughZK() throws Exception {
 
         // Launch a TSO publishing the address in HA...
         TSOServerConfig config = new TSOServerConfig();
-        config.setMaxItems(1000);
+        config.setConflictMapSize(1000);
         config.setPort(tsoPortForTest);
         config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
         injector = Guice.createInjector(new TSOMockModule(config));
@@ -191,7 +191,7 @@ public void testSuccessOfTSOClientReconnectionsToARestartedTSOWithZKPublishing()
 
         // Start a TSO with HA...
         TSOServerConfig config = new TSOServerConfig();
-        config.setMaxItems(1000);
+        config.setConflictMapSize(1000);
         config.setPort(tsoPortForTest);
         config.setLeaseModule(new HALeaseManagementModule(1000, TSO_LEASE_PATH, CURRENT_TSO_PATH, zkClusterForTest, "omid"));
         injector = Guice.createInjector(new TSOMockModule(config));

tso-server/src/test/java/org/apache/omid/tso/client/TestTSOClientRequestAndResponseBehaviours.java

@@ -71,7 +71,7 @@ public class TestTSOClientRequestAndResponseBehaviours {
     public void beforeMethod() throws Exception {
 
         TSOServerConfig tsoConfig = new TSOServerConfig();
-        tsoConfig.setMaxItems(1000);
+        tsoConfig.setConflictMapSize(1000);
         tsoConfig.setPort(TSO_SERVER_PORT);
         tsoConfig.setNumConcurrentCTWriters(2);
         Module tsoServerMockModule = new TSOMockModule(tsoConfig);

tso-server/src/test/resources/test-omid.yml

@@ -1,10 +1,7 @@
 # Non High Availability configuration with HBase for everything
 
-principal: sieve_omid0@YGRID.YAHOO.COM
-keytab: /home/sieve_omid0/sieve_omid0.prod.headless.keytab
-
 port: 54758
-maxItems: 100000000
+conflictMapSize: 100000000
 numConcurrentCTWriters: 2
 batchSizePerCTWriter: 500
 batchPersistTimeoutInMs: 100