Skip to content
Permalink
Browse files
add computer-k8s and computer-k8s-operator (#56)
  • Loading branch information
coderzc committed Aug 3, 2021
1 parent 9a53ac0 commit 7f82d964cc107b724a527c0455b186f80698c563
Showing 91 changed files with 11,535 additions and 22 deletions.
@@ -16,14 +16,22 @@ on:

jobs:
computer-ci:
runs-on: ubuntu-16.04
runs-on: ubuntu-20.04
steps:
- name: Install JDK 8
uses: actions/setup-java@v2
with:
java-version: '8'
distribution: 'adopt'

- name: Setup Minikube-Kubernetes
uses: manusa/actions-setup-minikube@v2.4.2
with:
minikube version: v1.21.0
kubernetes version: v1.20.1
- name: Interact with the cluster
run: kubectl get nodes

- name: Cache Maven packages
uses: actions/cache@v2
with:
@@ -37,7 +45,7 @@ jobs:
fetch-depth: 2

- name: Compile
run: mvn compile -Dmaven.javadoc.skip=true | grep -v "Downloading\|Downloaded"
run: mvn clean compile -Dmaven.javadoc.skip=true | grep -v "Downloading\|Downloaded"

- name: Prepare env and service
env:
@@ -58,4 +66,3 @@ jobs:
uses: codecov/codecov-action@v1
with:
file: target/site/jacoco/jacoco.xml

@@ -4,7 +4,7 @@
[![Build Status](https://travis-ci.org/hugegraph-ee/hugegraph-computer.svg?branch=master)](https://travis-ci.org/hugegraph-ee/hugegraph-computer)
[![codecov](https://codecov.io/gh/hugegraph-ee/hugegraph-computer/branch/master/graph/badge.svg)](https://codecov.io/gh/hugegraph-ee/hugegraph-computer)

hugegraph-computer is a distributed graph processing system for hugegraph. It is an implementaion of [Pregel](https://kowshik.github.io/JPregel/pregel_paper.pdf). It runs on YARN framework.
hugegraph-computer is a distributed graph processing system for hugegraph. It is an implementaion of [Pregel](https://kowshik.github.io/JPregel/pregel_paper.pdf). It runs on Kubernetes or YARN framework.

## Features

@@ -0,0 +1,9 @@
FROM hugegraph/hugegraph-computer-framework:latest

LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"

ARG jarFilePath="/opt/jars/computer-algorithm-based.jar"

COPY target/computer-algorithm-*.jar ${jarFilePath}

ENV JAR_FILE_PATH=${jarFilePath}
@@ -19,7 +19,7 @@ cd ../
rm -rf hugegraph
tar -zxf hugegraph-*.tar.gz

cd hugegraph-*
cd "$(find hugegraph-* | head -1)"
bin/init-store.sh || exit 1
bin/start-hugegraph.sh || exit 1
cd ../
@@ -5,4 +5,8 @@ set -ev
TRAVIS_DIR=`dirname $0`
echo "Starting etcd..."
chmod a+x ./computer-dist/src/main/resources/etcd
./computer-dist/src/main/resources/etcd &
./computer-dist/src/main/resources/etcd --name etcd-test \
--initial-advertise-peer-urls http://localhost:2580 \
--listen-peer-urls http://localhost:2580 \
--advertise-client-urls http://localhost:2579 \
--listen-client-urls http://localhost:2579 &
@@ -12,6 +12,9 @@
<artifactId>computer-driver</artifactId>

<dependencies>

<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-common</artifactId>
</dependency>
</dependencies>
</project>
@@ -17,24 +17,23 @@
* under the License.
*/

package com.baidu.hugegraph.computer.core.driver;
package com.baidu.hugegraph.computer.driver;

import java.io.Closeable;
import java.io.InputStream;
import java.util.List;
import java.util.Map;

import com.baidu.hugegraph.computer.core.graph.SuperstepStat;

public interface ComputerDriver {
public interface ComputerDriver extends Closeable {

/**
* This method is called when create algorithm, the user will upload a jar.
* @param input The input from which to create a jar.
* @param algorithmName The algorithm name is unique. If the jar
* correspond to the algorithm name is exist, it
* will overwrite the previous jar.
* @param input The input stream from which to create a jar.
*/
void uploadAlgorithmJar(InputStream input, String algorithmName);
void uploadAlgorithmJar(String algorithmName, InputStream input);

/**
* Submit the algorithm to the cluster, the parameters are in config.
@@ -58,8 +57,7 @@ public interface ComputerDriver {
* @param params reserved for other parameters in addition to jobId used
* to wait job.
*/
void waitJob(String jobId,
Map<String, String> params,
void waitJob(String jobId, Map<String, String> params,
JobObserver observer);

/**
@@ -0,0 +1,57 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/

package com.baidu.hugegraph.computer.driver;

/**
* The base class for all exception types
*/
public class ComputerDriverException extends RuntimeException {

private static final long serialVersionUID = -4048732559246459545L;

public ComputerDriverException(String message) {
super(message);
}

public ComputerDriverException(String message, Throwable cause) {
super(message, cause);
}

public ComputerDriverException(String message, Object... args) {
super(String.format(message, args));
}

public ComputerDriverException(String message, Throwable cause,
Object... args) {
super(String.format(message, args), cause);
}

public Throwable rootCause() {
return rootCause(this);
}

public static Throwable rootCause(Throwable e) {
Throwable cause = e;
while (cause.getCause() != null) {
cause = cause.getCause();
}
return cause;
}
}
@@ -0,0 +1,92 @@
/*
* Copyright 2017 HugeGraph Authors
*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/

package com.baidu.hugegraph.computer.driver;

import java.util.Objects;

public class DefaultJobState implements JobState {

private int superstep;
private int maxSuperstep;
private SuperstepStat lastSuperstepStat;
private JobStatus jobStatus;

public DefaultJobState superstep(int superstep) {
this.superstep = superstep;
return this;
}

public DefaultJobState maxSuperstep(int maxSuperstep) {
this.maxSuperstep = maxSuperstep;
return this;
}

public DefaultJobState lastSuperstepStat(SuperstepStat lastSuperstepStat) {
this.lastSuperstepStat = lastSuperstepStat;
return this;
}

public DefaultJobState jobStatus(JobStatus jobStatus) {
this.jobStatus = jobStatus;
return this;
}

@Override
public int superstep() {
return this.superstep;
}

@Override
public int maxSuperstep() {
return this.maxSuperstep;
}

@Override
public SuperstepStat lastSuperstepStat() {
return this.lastSuperstepStat;
}

@Override
public JobStatus jobStatus() {
return this.jobStatus;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof DefaultJobState)) {
return false;
}
DefaultJobState jobState = (DefaultJobState) o;
return this.superstep == jobState.superstep &&
this.maxSuperstep == jobState.maxSuperstep &&
Objects.equals(this.lastSuperstepStat,
jobState.lastSuperstepStat) &&
this.jobStatus == jobState.jobStatus;
}

@Override
public int hashCode() {
return Objects.hash(this.superstep, this.maxSuperstep,
this.lastSuperstepStat, this.jobStatus);
}
}
@@ -17,7 +17,7 @@
* under the License.
*/

package com.baidu.hugegraph.computer.core.driver;
package com.baidu.hugegraph.computer.driver;

public interface JobObserver {

@@ -17,9 +17,8 @@
* under the License.
*/

package com.baidu.hugegraph.computer.core.driver;
package com.baidu.hugegraph.computer.driver;

import com.baidu.hugegraph.computer.core.graph.SuperstepStat;

public interface JobState {

@@ -17,7 +17,9 @@
* under the License.
*/

package com.baidu.hugegraph.computer.core.driver;
package com.baidu.hugegraph.computer.driver;

import java.util.Objects;

public enum JobStatus {

@@ -30,4 +32,10 @@ public enum JobStatus {
public static boolean finished(JobStatus status) {
return status == CANCELLED || status == FAILED || status == SUCCEEDED;
}

public static boolean finished(String status) {
return Objects.equals(status, CANCELLED.name()) ||
Objects.equals(status, FAILED.name()) ||
Objects.equals(status, SUCCEEDED.name());
}
}

0 comments on commit 7f82d96

Please sign in to comment.