Skip to content
Permalink
Browse files
GIRAPH-1188
closes #70
  • Loading branch information
Yuksel Akinci authored and dlogothetis committed May 15, 2018
1 parent 345f3db commit b2d77411ae4a43ddd2386c4a1db5e0191790acf6
Show file tree
Hide file tree
Showing 10 changed files with 951 additions and 9 deletions.
@@ -1,3 +1,20 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.giraph.block_app.framework.no_vtx;

import java.util.Iterator;
@@ -25,6 +25,7 @@
import org.apache.giraph.partition.GraphPartitionerFactory;
import org.apache.giraph.utils.CheckpointingUtils;
import org.apache.giraph.worker.WorkerInfo;
import org.apache.giraph.writable.kryo.GiraphClassResolver;
import org.apache.giraph.zk.BspEvent;
import org.apache.giraph.zk.PredicateLock;
import org.apache.giraph.zk.ZooKeeperExt;
@@ -81,7 +82,9 @@
/** Input splits all done node*/
public static final String INPUT_SPLITS_ALL_DONE_NODE =
"/_inputSplitsAllDone";

/** Directory to store kryo className-ID assignment */
public static final String KRYO_REGISTERED_CLASS_DIR =
"/_kryo";
/** Directory of attempts of this application */
public static final String APPLICATION_ATTEMPTS_DIR =
"/_applicationAttemptsDir";
@@ -155,6 +158,8 @@
protected final String haltComputationPath;
/** Path where memory observer stores data */
protected final String memoryObserverPath;
/** Kryo className-ID mapping directory */
protected final String kryoRegisteredClassPath;
/** Private ZooKeeper instance that implements the service */
private final ZooKeeperExt zk;
/** Has the Connection occurred? */
@@ -250,7 +255,7 @@ public BspService(
inputSplitsAllDonePath = basePath + INPUT_SPLITS_ALL_DONE_NODE;
applicationAttemptsPath = basePath + APPLICATION_ATTEMPTS_DIR;
cleanedUpPath = basePath + CLEANED_UP_DIR;

kryoRegisteredClassPath = basePath + KRYO_REGISTERED_CLASS_DIR;


String restartJobId = RESTART_JOB_ID.get(conf);
@@ -289,6 +294,11 @@ public BspService(
throw new RuntimeException(e);
}

boolean disableGiraphResolver =
GiraphConstants.DISABLE_GIRAPH_CLASS_RESOLVER.get(conf);
if (!disableGiraphResolver) {
GiraphClassResolver.setZookeeperInfo(zk, kryoRegisteredClassPath);
}
this.taskId = (int) getApplicationAttempt() * conf.getMaxWorkers() +
conf.getTaskPartition();
this.hostnameTaskId = hostname + "_" + getTaskId();
@@ -1292,5 +1292,15 @@ public interface GiraphConstants {
/** Number of supersteps job will run for */
IntConfOption SUPERSTEP_COUNT = new IntConfOption("giraph.numSupersteps", -1,
"Number of supersteps job will run for");

/** Whether to disable GiraphClassResolver which is an efficient
* implementation of kryo class resolver. By default this resolver is used by
* KryoSimpleWritable and KryoSimpleWrapper, and can be disabled with this
* option */
BooleanConfOption DISABLE_GIRAPH_CLASS_RESOLVER =
new BooleanConfOption("giraph.disableGiraphClassResolver", false,
"Disables GiraphClassResolver, which is a custom implementation " +
"of kryo class resolver that avoids writing class names to the " +
"underlying stream for faster serialization.");
}
// CHECKSTYLE: resume InterfaceIsTypeCheck
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.giraph.writable.kryo;

/**
* Boxed interface
* @param <T>
*/
public interface Boxed<T> {
/**
* Gets the boxed value.
* @return Boxed object.
*/
T get();

/**
* Sets the boxed value.
* @param value Value
*/
void set(T value);
}

0 comments on commit b2d7741

Please sign in to comment.