Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Hmusum/upgrade to zookeeper 3.5 #10654

Merged
merged 6 commits into from Sep 17, 2019
Merged

Hmusum/upgrade to zookeeper 3.5 #10654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
424edf5
WIP: Upgrade to zookeeper 3.5
Sep 6, 2019
fcfed68
Fix unit tests
Sep 6, 2019
ddf2e1a
Fix
Sep 8, 2019
384f6fb
Don't use import wildcards.
gjoranv Sep 9, 2019
07a51de
Fix error from rebasing onto master: server numbering starts at 0.
gjoranv Sep 10, 2019
27fad86
Add spotbugs dep to clustercontroller-core.
gjoranv Sep 10, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 7 additions & 0 deletions clustercontroller-core/pom.xml
View file
Open in desktop
Expand Up @@ -65,6 +65,13 @@
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<!-- Not used by this module, but compilation fails without it because zookeeper uses these annotations.
Provided scoped here to avoid dependents getting it transitively. -->
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
Expand Down
26 changes: 17 additions & 9 deletions ...core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java
View file
Open in desktop
@@ -1,25 +1,33 @@
// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.clustercontroller.core.database;

import com.yahoo.log.LogLevel;
import com.yahoo.vdslib.state.Node;
import com.yahoo.vdslib.state.NodeState;
import com.yahoo.vdslib.state.State;
import com.yahoo.vespa.clustercontroller.core.AnnotatedClusterState;
import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle;
import com.yahoo.vespa.clustercontroller.core.ContentCluster;
import com.yahoo.vespa.clustercontroller.core.rpc.EnvelopedClusterStateBundleCodec;
import com.yahoo.vespa.clustercontroller.core.rpc.SlimeClusterStateBundleCodec;
import org.apache.zookeeper.*;
import org.apache.zookeeper.data.Stat;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.ACL;
import com.yahoo.log.LogLevel;
import com.yahoo.vdslib.state.NodeState;
import com.yahoo.vdslib.state.State;
import com.yahoo.vdslib.state.Node;
import org.apache.zookeeper.data.Stat;

import java.util.logging.Logger;
import java.util.*;
import java.io.IOException;
import java.io.StringWriter;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.TreeMap;
import java.util.logging.Logger;

public class ZooKeeperDatabase extends Database {

Expand Down
19 changes: 17 additions & 2 deletions parent/pom.xml
View file
Open in desktop
Expand Up @@ -505,14 +505,17 @@
<artifactId>commons-pool</artifactId>
<version>1.5.6</version>
</dependency>
<!-- Explicitly included to get Zookeeper version 3.4.14,

<!-- BEGIN zookeeper -->

<!-- Explicitly force Zookeeper version,
can be excluded if you want the Zookeeper version
used by curator by default
-->
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.14</version>
<version>${zookeeper.version}</version>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
Expand All @@ -524,6 +527,17 @@
<artifactId>curator-test</artifactId>
<version>${curator.version}</version>
</dependency>
<dependency>
<!-- Needed by zookeeper, which has an optional dependency.
Version must be the same as the one zk uses - check the zk pom file. -->
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-annotations</artifactId>
<!-- No version property, as we don't want maven-dependency-plugin to alert about newer versions. -->
<version>3.1.9</version>
</dependency>

<!-- END zookeeper -->

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
Expand Down Expand Up @@ -789,6 +803,7 @@
<junit.version>5.4.2</junit.version>
<prometheus.client.version>0.6.0</prometheus.client.version>
<protobuf.version>3.7.0</protobuf.version>
<zookeeper.version>3.5.5</zookeeper.version>
</properties>

</project>
12 changes: 7 additions & 5 deletions zkfacade/src/main/java/com/yahoo/vespa/zookeeper/ZooKeeperServer.java
View file
Open in desktop
Expand Up @@ -66,10 +66,12 @@ private String transformConfigToString(ZookeeperServerConfig config) {
sb.append("clientPort=").append(config.clientPort()).append("\n");
sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n");
sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n");
// See http://zookeeper.apache.org/doc/r3.4.13/zookeeperAdmin.html#sc_zkCommands
// Includes all available commands in 3.4, except 'wchc' and 'wchp'
// Mandatory when using ZooKeeper 3.5
sb.append("4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n");
// See http://zookeeper.apache.org/doc/r3.5.5/zookeeperAdmin.html#sc_zkCommands
// Includes all available commands in 3.5, except 'wchc' and 'wchp'
sb.append("4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n");
sb.append("admin.enableServer=false").append("\n");
// Need NettyServerCnxnFactory to be able to use TLS for communication
sb.append("serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory").append("\n");
ensureThisServerIsRepresented(config.myid(), config.server());
config.server().forEach(server -> addServerToCfg(sb, server));
return sb.toString();
Expand Down Expand Up @@ -113,7 +115,7 @@ private void shutdown() {
public void run() {
System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true");
String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())};
log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config: " + args[0]);
log.log(LogLevel.DEBUG, "Starting ZooKeeper server with config file " + args[0]);
log.log(LogLevel.INFO, "Trying to establish ZooKeeper quorum (from " + zookeeperServerHostnames(zookeeperServerConfig) + ")");
org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args);
}
Expand Down
8 changes: 6 additions & 2 deletions zkfacade/src/test/java/com/yahoo/vespa/zookeeper/ZooKeeperServerTest.java
View file
Open in desktop
Expand Up @@ -110,7 +110,9 @@ private void validateConfigFileSingleHost(File cfgFile) throws IOException {
"clientPort=2181\n" +
"autopurge.purgeInterval=1\n" +
"autopurge.snapRetainCount=15\n" +
"4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
"4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
"admin.enableServer=false\n" +
"serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory\n" +
"server.0=foo:321:123\n";
validateConfigFile(cfgFile, expected);
}
Expand All @@ -126,7 +128,9 @@ private void validateConfigFileMultipleHosts(File cfgFile) throws IOException {
"clientPort=2181\n" +
"autopurge.purgeInterval=1\n" +
"autopurge.snapRetainCount=15\n" +
"4lw.commands.whitelist=conf,cons,crst,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
"4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs\n" +
"admin.enableServer=false\n" +
"serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory\n" +
"server.0=foo:321:123\n" +
"server.1=bar:432:234\n" +
"server.2=baz:543:345\n";
Expand Down