Skip to content
This repository has been archived by the owner on Oct 30, 2023. It is now read-only.

Commit

Permalink
GIRAPH-991 Update versions of guava and swift
Browse files Browse the repository at this point in the history
Summary: Changes in WorkerProgress are needed because newer version of thrift requires thrift classes to be final.

Test Plan: mvn clean verify
+ run a bunch of test jobs

Reviewers: maja.kabiljo

Differential Revision: https://reviews.facebook.net/D32757
  • Loading branch information
Sergey Edunov committed Feb 4, 2015
1 parent 931569d commit 63b8c41
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 55 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG
@@ -1,6 +1,8 @@
Giraph Change Log

Release 1.2.0 - unreleased
GIRAPH-991: Update versions of guava and swift (edunov)

GIRAPH-987: Improve naming for ReduceOperation (ikabiljo via majakabiljo)

GIRAPH-986: Add no-arg constructor to BasicSet (ikabiljo via edunov)
Expand Down
Expand Up @@ -18,22 +18,21 @@

package org.apache.giraph.job;

import com.google.common.collect.Iterables;
import org.apache.giraph.conf.FloatConfOption;
import org.apache.giraph.worker.WorkerProgress;
import org.apache.giraph.worker.WorkerProgressStats;
import org.apache.hadoop.conf.Configuration;

import com.google.common.collect.Iterables;

import java.text.DecimalFormat;

import javax.annotation.concurrent.NotThreadSafe;
import java.text.DecimalFormat;

/**
* Class which combines multiple workers' progresses to get overall
* application progress
*/
@NotThreadSafe
public class CombinedWorkerProgress extends WorkerProgress {
public class CombinedWorkerProgress extends WorkerProgressStats {
/** Decimal format which rounds numbers to two decimal places */
public static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("#.##");
/**
Expand Down
Expand Up @@ -18,10 +18,9 @@

package org.apache.giraph.worker;

import org.apache.giraph.utils.MemoryUtils;

import com.facebook.swift.codec.ThriftField;
import com.facebook.swift.codec.ThriftStruct;
import org.apache.giraph.utils.MemoryUtils;

import javax.annotation.concurrent.ThreadSafe;

Expand All @@ -31,56 +30,16 @@
*/
@ThreadSafe
@ThriftStruct
public class WorkerProgress {
public final class WorkerProgress extends WorkerProgressStats {
/** Singleton instance for everyone to use */
private static final WorkerProgress INSTANCE = new WorkerProgress();

/** Superstep which worker is executing, Long.MAX_VALUE if it's output */
protected long currentSuperstep = -1;

/** How many vertices were loaded until now */
protected long verticesLoaded = 0;
/** How many vertex input splits were loaded until now */
protected int vertexInputSplitsLoaded = 0;
/** Whether worker finished loading vertices */
protected boolean loadingVerticesDone = false;
/** How many edges were loaded */
protected long edgesLoaded = 0;
/** How many edge input splits were loaded until now */
protected int edgeInputSplitsLoaded = 0;
/** Whether worker finished loading edges until now */
protected boolean loadingEdgesDone = false;

/** How many vertices are there to compute in current superstep */
protected long verticesToCompute = 0;
/** How many vertices were computed in current superstep until now */
protected long verticesComputed = 0;
/** How many partitions are there to compute in current superstep */
protected int partitionsToCompute = 0;
/** How many partitions were computed in current superstep until now */
protected int partitionsComputed = 0;

/** Whether all compute supersteps are done */
protected boolean computationDone = false;

/** How many vertices are there to store */
protected long verticesToStore = 0;
/** How many vertices were stored until now */
protected long verticesStored = 0;
/** How many partitions are there to store */
protected int partitionsToStore = 0;
/** How many partitions were stored until now */
protected int partitionsStored = 0;
/** Whether worker finished storing data */
protected boolean storingDone = false;

/** Id of the mapper */
protected int taskId;

/** Free memory */
protected double freeMemoryMB;
/** Fraction of memory that's free */
protected double freeMemoryFraction;
/**
* Public constructor for thrift to create us.
* Please use WorkerProgress.get() to get the static instance.
*/
public WorkerProgress() {
}

/**
* Get singleton instance of WorkerProgress.
Expand Down
@@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.giraph.worker;

import javax.annotation.concurrent.NotThreadSafe;

/**
* Stats about a worker's progress
*/
@NotThreadSafe
public class WorkerProgressStats {
/** Superstep which worker is executing, Long.MAX_VALUE if it's output */
protected long currentSuperstep = -1;

/** How many vertices were loaded until now */
protected long verticesLoaded = 0;
/** How many vertex input splits were loaded until now */
protected int vertexInputSplitsLoaded = 0;
/** Whether worker finished loading vertices */
protected boolean loadingVerticesDone = false;
/** How many edges were loaded */
protected long edgesLoaded = 0;
/** How many edge input splits were loaded until now */
protected int edgeInputSplitsLoaded = 0;
/** Whether worker finished loading edges until now */
protected boolean loadingEdgesDone = false;

/** How many vertices are there to compute in current superstep */
protected long verticesToCompute = 0;
/** How many vertices were computed in current superstep until now */
protected long verticesComputed = 0;
/** How many partitions are there to compute in current superstep */
protected int partitionsToCompute = 0;
/** How many partitions were computed in current superstep until now */
protected int partitionsComputed = 0;

/** Whether all compute supersteps are done */
protected boolean computationDone = false;

/** How many vertices are there to store */
protected long verticesToStore = 0;
/** How many vertices were stored until now */
protected long verticesStored = 0;
/** How many partitions are there to store */
protected int partitionsToStore = 0;
/** How many partitions were stored until now */
protected int partitionsStored = 0;
/** Whether worker finished storing data */
protected boolean storingDone = false;

/** Id of the mapper */
protected int taskId;

/** Free memory */
protected double freeMemoryMB;
/** Fraction of memory that's free */
protected double freeMemoryFraction;

public boolean isInputSuperstep() {
return currentSuperstep == -1;
}

public boolean isComputeSuperstep() {
return currentSuperstep >= 0 && currentSuperstep < Long.MAX_VALUE;
}

public boolean isOutputSuperstep() {
return currentSuperstep == Long.MAX_VALUE;
}
}
2 changes: 1 addition & 1 deletion pom.xml
Expand Up @@ -291,7 +291,7 @@ under the License.
<dep.commons-logging.version>1.1.1</dep.commons-logging.version>
<dep.commons-io.version>2.1</dep.commons-io.version>
<dep.commons-net.version>3.1</dep.commons-net.version>
<dep.facebook-swift.version>0.13.1</dep.facebook-swift.version>
<dep.facebook-swift.version>0.14.0</dep.facebook-swift.version>
<dep.fasterxml-jackson.version>2.1.2</dep.fasterxml-jackson.version>
<dep.fastutil.version>6.5.4</dep.fastutil.version>
<dep.google.findbugs.version>2.0.2</dep.google.findbugs.version>
Expand Down

0 comments on commit 63b8c41

Please sign in to comment.