Skip to content
Permalink
Browse files
update java version to 11 (#170)
  • Loading branch information
coderzc committed Jan 18, 2022
1 parent d890a24 commit 18bedd260feac60b07596d39a58eaf0d3194b5cb
Showing 13 changed files with 62 additions and 48 deletions.
@@ -22,7 +22,7 @@ jobs:
uses: actions/setup-java@v2
with:
java-version: '8'
distribution: 'adopt'
distribution: 'zulu'

- name: Setup Python3
uses: actions/setup-python@v2
@@ -57,9 +57,6 @@ jobs:
with:
fetch-depth: 2

- name: Compile
run: mvn clean compile -Dmaven.javadoc.skip=true | grep -v "Downloading\|Downloaded"

- name: Prepare env and service
env:
TRAVIS_DIR: computer-dist/src/assembly/travis
@@ -69,6 +66,15 @@ jobs:
$TRAVIS_DIR/install-hugegraph-from-source.sh $COMMIT_ID | grep -v "Downloading\|Downloaded"
$TRAVIS_DIR/load-data-into-hugegraph.sh
- name: Install JDK 11
uses: actions/setup-java@v2
with:
java-version: '11'
distribution: 'zulu'

- name: Compile
run: mvn clean compile -Dmaven.javadoc.skip=true | grep -v "Downloading\|Downloaded"

- name: Integrate test
run: mvn test -P integrate-test

@@ -36,11 +36,11 @@ jobs:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}

- name: Install JDK 8
- name: Install JDK 11
uses: actions/setup-java@v2
with:
java-version: '8'
distribution: 'adopt'
java-version: '11'
distribution: 'zulu'

- name: Cache Maven packages
uses: actions/cache@v2
@@ -8,10 +8,10 @@ jobs:
build:
runs-on: ubuntu-20.04
steps:
- name: Install JDK 8
- name: Install JDK 11
uses: actions/setup-java@v2
with:
java-version: '8'
java-version: '11'
distribution: 'adopt'

- name: Cache Maven packages
@@ -638,15 +638,15 @@ public static synchronized ComputerOptions instance() {
"transport.server_threads",
"The number of transport threads for server.",
positiveInt(),
TransportConf.NUMBER_CPU_CORES
TransportConf.DEFAULT_THREADS
);

public static final ConfigOption<Integer> TRANSPORT_CLIENT_THREADS =
new ConfigOption<>(
"transport.client_threads",
"The number of transport threads for client.",
positiveInt(),
TransportConf.NUMBER_CPU_CORES
TransportConf.DEFAULT_THREADS
);

public static final ConfigOption<Class<?>> TRANSPORT_PROVIDER_CLASS =
@@ -35,10 +35,10 @@ public class TransportConf {
"transport-netty-server";
public static final String CLIENT_THREAD_GROUP_NAME =
"transport-netty-client";
public static final int NUMBER_CPU_CORES =
Runtime.getRuntime().availableProcessors();
private final Config config;

public static final int DEFAULT_THREADS = 4;

public static TransportConf wrapConfig(Config config) {
return new TransportConf(config);
}
@@ -72,11 +72,7 @@ public int clientThreads() {
}

private int maxTransportThreads() {
Integer workerCount = this.config
.get(ComputerOptions.JOB_WORKERS_COUNT);
Integer partitions = this.config
.get(ComputerOptions.JOB_PARTITIONS_COUNT);
return partitions / workerCount + 1;
return this.config.get(ComputerOptions.JOB_WORKERS_COUNT);
}

public TransportProvider transportProvider() {
@@ -1,6 +1,7 @@
FROM openjdk:8-jre
FROM openjdk:11-jre
LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=2 -XshowSettings:vm"
# use ParallelGC which is more friendly to olap system
ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseParallelGC -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm"
COPY . /etc/local/hugegraph-computer
WORKDIR /etc/local/hugegraph-computer
RUN apt-get update && apt-get -y install gettext-base && apt-get -y install wget
@@ -164,6 +164,17 @@ else
JAVA="$JAVA_HOME/bin/java -server"
fi

# Set up count of cpu if it unspecified from k8s drive,
# avoid `Runtime.getRuntime().availableProcessors()` always return 1
if [[ "${DRIVE}" = "${K8S_DRIVE}" && -z "${CPU_LIMIT}" ]]; then
PROCESSOR_COUNT="$(cat /proc/cpuinfo | grep "processor" | wc -l)"
let MAX_PROCESSOR_COUNT=8
if [[ ${PROCESSOR_COUNT} -gt ${MAX_PROCESSOR_COUNT} ]]; then
PROCESSOR_COUNT="$MAX_PROCESSOR_COUNT"
fi
JAVA_OPTS="${JAVA_OPTS} -XX:ActiveProcessorCount=${PROCESSOR_COUNT}"
fi

if [ ! -a "${CONF_DIR}" ];then
mkdir -p "${CONF_DIR}"
fi
@@ -185,9 +196,9 @@ fi
MAIN_CLASS=com.baidu.hugegraph.computer.dist.HugeGraphComputer

if [ "${LOG4j_CONF}" != "" ]; then
exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JVM_OPTIONS} \
exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JAVA_OPTS} ${JVM_OPTIONS} \
-cp "${CP}" ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
else
exec ${JAVA} -Dname="hugegraph-computer" ${JVM_OPTIONS} -cp "${CP}" \
exec ${JAVA} -Dname="hugegraph-computer" ${JAVA_OPTS} ${JVM_OPTIONS} -cp "${CP}" \
${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
fi
@@ -1,7 +1,5 @@
FROM openjdk:8-jre-alpine
FROM openjdk:11-jre
LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"

ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=2 -XshowSettings:vm"
WORKDIR /opt/app
COPY target/hugegraph-computer-operator-*.jar hugegraph-computer-operator.jar
ENTRYPOINT ["java", "-jar", "hugegraph-computer-operator.jar"]
@@ -52,12 +52,14 @@ spec:
image: hugegraph/hugegraph-builtin-algorithms:latest # algorithm image url
jarFile: /opt/jars/hugegraph-builtin-algorithms.jar
pullPolicy: Always
workerInstances: 50
workerCpu: "4"
workerMemory: "4Gi"
workerInstances: 5
computerConf:
job.partitions_count: "20"
algorithm.params_class: com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams
hugegraph.url: http://${hugegraph-server-host}:${hugegraph-server-port} # hugegraph server url
job.partitions_count: "200"
hugegraph.name: "hugegraph"
hugegraph.name: hugegraph
EOF
```

@@ -85,7 +85,6 @@ public class ComputerJobDeployer {
private static final String RPC_PORT_NAME = "rpc-port";
private static final int DEFAULT_TRANSPORT_PORT = 8099;
private static final int DEFAULT_RPC_PORT = 8090;
private static final int DEFAULT_TRANSPORT_THREADS = 8;
private static final String COMPUTER_CONFIG_MAP_VOLUME =
"computer-config-map-volume";

@@ -198,19 +197,6 @@ private Set<ContainerPort> handleConfig(ComputerJobSpec spec) {
config.put(ComputerOptions.RPC_SERVER_PORT.name(), rpcPort);
}

/*
Set a default number of transport threads,
if the number of CPU quantity of the worker is not specified
*/
if (spec.getWorkerCpu() == null) {
String defaultThreads = String.valueOf(DEFAULT_TRANSPORT_THREADS);

config.putIfAbsent(ComputerOptions.TRANSPORT_CLIENT_THREADS.name(),
defaultThreads);
config.putIfAbsent(ComputerOptions.TRANSPORT_SERVER_THREADS.name(),
defaultThreads);
}

ContainerPort transportContainerPort = new ContainerPortBuilder()
.withName(TRANSPORT_PORT_NAME)
.withContainerPort(Integer.valueOf(transportPort))
@@ -471,6 +457,22 @@ private Container getContainer(String name, ComputerJobSpec spec,
memory = spec.getWorkerMemory();
}

if (cpu != null) {
EnvVar cpuLimit = new EnvVarBuilder()
.withName(Constants.ENV_CPU_LIMIT)
.withValue(cpu.toString())
.build();
envVars.add(cpuLimit);
}

if (memory != null) {
EnvVar memoryLimit = new EnvVarBuilder()
.withName(Constants.ENV_MEMORY_LIMIT)
.withValue(memory.toString())
.build();
envVars.add(memoryLimit);
}

List<VolumeMount> volumeMounts = spec.getVolumeMounts();
if (volumeMounts == null) {
volumeMounts = new ArrayList<>();
@@ -42,6 +42,8 @@ public class Constants {
public static final String ENV_JAR_FILE_PATH = "JAR_FILE_PATH";
public static final String ENV_JOB_JAR_URI = "JOB_JAR_URI";
public static final String ENV_JVM_OPTIONS = "JVM_OPTIONS";
public static final String ENV_CPU_LIMIT = "CPU_LIMIT";
public static final String ENV_MEMORY_LIMIT = "MEMORY_LIMIT";

public static final String CONFIG_DIR = "/opt/hugegraph-computer/conf";
public static final String COMPUTER_CONF_FILE = "computer.properties";
@@ -130,14 +130,10 @@ public void testGet() {

Assert.assertThrows(IndexOutOfBoundsException.class, () -> {
value1.get(3);
}, e -> {
Assert.assertContains("Index: 3, Size: 3", e.getMessage());
});

Assert.assertThrows(IndexOutOfBoundsException.class, () -> {
value2.get(3);
}, e -> {
Assert.assertContains("Index: 3, Size: 2", e.getMessage());
});
}

@@ -42,8 +42,8 @@
<top.level.dir>${project.basedir}/..</top.level.dir>
<release.name>hugegraph-computer</release.name>
<final.name>${release.name}-${project.version}</final.name>
<compiler.source>1.8</compiler.source>
<compiler.target>1.8</compiler.target>
<compiler.source>11</compiler.source>
<compiler.target>11</compiler.target>
<shell-executable>bash</shell-executable>
<etcd.version>0.5.4</etcd.version>
<hadoop-version>3.1.2</hadoop-version>

0 comments on commit 18bedd2

Please sign in to comment.