Skip to content
This repository has been archived by the owner on Oct 30, 2023. It is now read-only.

Commit

Permalink
GIRAPH-20 Move temporary test files from the project directory
Browse files Browse the repository at this point in the history
git-svn-id: https://svn.apache.org/repos/asf/incubator/giraph/trunk@1336743 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information
asubmissions committed May 10, 2012
1 parent 27c3bfb commit 79962a3
Show file tree
Hide file tree
Showing 16 changed files with 687 additions and 600 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG
Expand Up @@ -2,6 +2,8 @@ Giraph Change Log

Release 0.2.0 - unreleased

GIRAPH-20. Move temporary test files from the project directory. (ssc)

GIRAPH-37. Implement Netty-backed IPC. (aching)

GIRAPH-184. Upgrade to junit4. (Devaraj K via jghoman)
Expand Down
Expand Up @@ -135,12 +135,9 @@ public void postApplication() {
@Override
public void preSuperstep() {

LongSumAggregator sumAggreg =
(LongSumAggregator) getAggregator("sum");
MinAggregator minAggreg =
(MinAggregator) getAggregator("min");
MaxAggregator maxAggreg =
(MaxAggregator) getAggregator("max");
LongSumAggregator sumAggreg = (LongSumAggregator) getAggregator("sum");
MinAggregator minAggreg = (MinAggregator) getAggregator("min");
MaxAggregator maxAggreg = (MaxAggregator) getAggregator("max");

if (getSuperstep() >= 3) {
LOG.info("aggregatedNumVertices=" +
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/org/apache/giraph/graph/GraphMapper.java
Expand Up @@ -130,8 +130,7 @@ class OverrideExceptionHandler implements Thread.UncaughtExceptionHandler {
public void uncaughtException(Thread t, Throwable e) {
LOG.fatal(
"uncaughtException: OverrideExceptionHandler on thread " +
t.getName() + ", msg = " + e.getMessage() +
", exiting...", e);
t.getName() + ", msg = " + e.getMessage() + ", exiting...", e);
System.exit(1);
}
}
Expand Down
12 changes: 7 additions & 5 deletions src/main/java/org/apache/giraph/graph/TextAggregatorWriter.java
Expand Up @@ -22,6 +22,7 @@
import java.util.Map;
import java.util.Map.Entry;

import com.google.common.base.Charsets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
Expand All @@ -42,6 +43,8 @@ public class TextAggregatorWriter implements AggregatorWriter {
public static final int NEVER = 0;
/** Signal for "write only the final values" frequency */
public static final int AT_THE_END = -1;
/** Signal for "write values in every superstep" frequency */
public static final int ALWAYS = -1;
/** The frequency of writing:
* - NEVER: never write, files aren't created at all
* - AT_THE_END: aggregators are written only when the computation is over
Expand Down Expand Up @@ -78,11 +81,10 @@ public final void writeAggregator(
Map<String, Aggregator<Writable>> aggregators,
long superstep) throws IOException {
if (shouldWrite(superstep)) {
for (Entry<String, Aggregator<Writable>> a:
aggregators.entrySet()) {
output.writeUTF(aggregatorToString(a.getKey(),
a.getValue(),
superstep));
for (Entry<String, Aggregator<Writable>> a: aggregators.entrySet()) {
byte[] bytes = aggregatorToString(a.getKey(), a.getValue(), superstep)
.getBytes(Charsets.UTF_8);
output.write(bytes, 0, bytes.length);
}
output.flush();
}
Expand Down
179 changes: 179 additions & 0 deletions src/main/java/org/apache/giraph/utils/FileUtils.java
@@ -0,0 +1,179 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.giraph.utils;

import com.google.common.base.Charsets;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.io.Writer;

/**
* Helper class for filesystem operations during testing
*/
public class FileUtils {

/**
* Utility class should not be instantiatable
*/
private FileUtils() {
}

/**
* Create a temporary folder that will be removed after the test.
*
* @param vertexClass Used for generating the folder name.
* @return File object for the directory.
*/
public static File createTestDir(Class<?> vertexClass)
throws IOException {
String systemTmpDir = System.getProperty("java.io.tmpdir");
long simpleRandomLong = (long) (Long.MAX_VALUE * Math.random());
File testTempDir = new File(systemTmpDir, "giraph-" +
vertexClass.getSimpleName() + '-' + simpleRandomLong);
if (!testTempDir.mkdir()) {
throw new IOException("Could not create " + testTempDir);
}
testTempDir.deleteOnExit();
return testTempDir;
}

/**
* Make a temporary file.
*
* @param parent Parent directory.
* @param name File name.
* @return File object to temporary file.
* @throws IOException
*/
public static File createTempFile(File parent, String name)
throws IOException {
return createTestTempFileOrDir(parent, name, false);
}

/**
* Make a temporary directory.
*
* @param parent Parent directory.
* @param name Directory name.
* @return File object to temporary file.
* @throws IOException
*/
public static File createTempDir(File parent, String name)
throws IOException {
File dir = createTestTempFileOrDir(parent, name, true);
dir.delete();
return dir;
}

/**
* Create a test temp file or directory.
*
* @param parent Parent directory
* @param name Name of file
* @param dir Is directory?
* @return File object
* @throws IOException
*/
public static File createTestTempFileOrDir(File parent, String name,
boolean dir) throws IOException {
File f = new File(parent, name);
f.deleteOnExit();
if (dir && !f.mkdirs()) {
throw new IOException("Could not make directory " + f);
}
return f;
}

/**
* Write lines to a file.
*
* @param file File to write lines to
* @param lines Strings written to the file
* @throws IOException
*/
public static void writeLines(File file, String... lines)
throws IOException {
Writer writer = Files.newWriter(file, Charsets.UTF_8);
try {
for (String line : lines) {
writer.write(line);
writer.write('\n');
}
} finally {
Closeables.closeQuietly(writer);
}
}

/**
* Recursively delete a directory
*
* @param dir Directory to delete
*/
public static void delete(File dir) {
if (dir != null) {
new DeletingVisitor().accept(dir);
}
}

/**
* Deletes files.
*/
private static class DeletingVisitor implements FileFilter {
@Override
public boolean accept(File f) {
if (!f.isFile()) {
f.listFiles(this);
}
f.delete();
return false;
}
}

/**
* Helper method to remove a path if it exists.
*
* @param conf Configuration to load FileSystem from
* @param path Path to remove
* @throws IOException
*/
public static void deletePath(Configuration conf, String path)
throws IOException {
deletePath(conf, new Path(path));
}

/**
* Helper method to remove a path if it exists.
*
* @param conf Configuration to load FileSystem from
* @param path Path to remove
* @throws IOException
*/
public static void deletePath(Configuration conf, Path path)
throws IOException {
FileSystem fs = FileSystem.get(conf);
fs.delete(path, true);
}
}

0 comments on commit 79962a3

Please sign in to comment.