Skip to content

Commit

Permalink
auto batch commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@hongsup2
hongsup2 committed Mar 1, 2013
1 parent fc332aa commit e04f700
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 22 deletions.
Binary file modified apps/output_giraph/lib/giraph-0.2-SNAPSHOT-for-hadoop-0.20.203.0-jar-with-dependencies.jar
View file
Binary file not shown.
14 changes: 7 additions & 7 deletions src/backend_giraph/gm_giraph_gen.cc
View file
Expand Up @@ -282,9 +282,9 @@ void gm_giraph_gen::do_generate_input_output_formats() {
// Parse edge values
//------------------------------------------------------------
Body_input.pushln("@Override");
sprintf(temp, "protected Iterable< Edge<%s, %s> > getEdges(String[] values) throws IOException {", vertex_id, edge_data);
sprintf(temp, "protected Map< %s, %s > getEdges(String[] values) throws IOException {", vertex_id, edge_data);
Body_input.pushln(temp);
sprintf(temp, "List< Edge<%s, %s> > edges = Lists.newLinkedList();", vertex_id, edge_data);
sprintf(temp, "Map< %s, %s > edges = new HashMap<%s, %s>();", vertex_id, edge_data, vertex_id, edge_data);
Body_input.pushln(temp);
int step ;
//if ((L2.size() == 0) && (OPTIONS.get_arg_bool(GMARGFLAG_GIRAPH_DUMMY_VALUE)))
Expand All @@ -296,13 +296,13 @@ void gm_giraph_gen::do_generate_input_output_formats() {
Body_input.pushln(temp);
// destination id
if (PREGEL_BE->get_lib()->is_node_type_int()) {
Body_input.pushln("IntWritable edgeId = new IntWritable(Integer.parseInt(values[i]));");
Body_input.pushln("IntWritable destId = new IntWritable(Integer.parseInt(values[i]));");
} else {
Body_input.pushln("LongWritable edgeId = new LongWritable(Long.parseLong(values[i]));");
Body_input.pushln("LongWritable destId = new LongWritable(Long.parseLong(values[i]));");
}
// edge properties
if (proc->find_info_bool(GPS_FLAG_USE_EDGE_PROP)) {
sprintf(temp, "edges.add(new DefaultEdge<%s, %s>(edgeId, new %s(", vertex_id, edge_data, edge_data);
sprintf(temp, "edges.put(destId, new %s(", edge_data);
Body_input.push(temp);
val = 1;
if (L2.size() > 0) {
Expand All @@ -314,11 +314,11 @@ void gm_giraph_gen::do_generate_input_output_formats() {
if (val != L2.size()) Body_input.push(", ");
}
}
Body_input.pushln(")));");
Body_input.pushln("));");
} else {
//if ((L2.size() == 0) && (OPTIONS.get_arg_bool(GMARGFLAG_GIRAPH_DUMMY_VALUE)))
// Body_input.pushln("// Ignoring dummy edge value");
sprintf(temp, "edges.add(new DefaultEdge<%s, %s>(edgeId, NullWritable.get()));", vertex_id, edge_data);
sprintf(temp, "edges.put(destId, NullWritable.get());", vertex_id, edge_data);
Body_input.pushln(temp);
}
Body_input.pushln("}");
Expand Down
21 changes: 10 additions & 11 deletions src/backend_giraph/gm_giraph_lib.cc
View file
Expand Up @@ -31,8 +31,6 @@ void gm_giraphlib::generate_headers_vertex(gm_code_writer& Body) {
Body.pushln("import org.apache.giraph.aggregators.*;");
Body.pushln("import org.apache.giraph.graph.*;");
Body.pushln("import org.apache.giraph.master.*;");
Body.pushln("import org.apache.giraph.vertex.*;");
Body.pushln("import org.apache.giraph.worker.*;");
Body.pushln("import org.apache.hadoop.io.*;");
Body.pushln("import org.apache.log4j.Logger;");
Body.NL();
Expand All @@ -48,9 +46,9 @@ void gm_giraphlib::generate_headers_main(gm_code_writer& Body) {
Body.pushln("import org.apache.commons.cli.HelpFormatter;");
Body.pushln("import org.apache.commons.cli.Options;");
Body.pushln("import org.apache.commons.cli.PosixParser;");
Body.pushln("import org.apache.giraph.conf.GiraphConfiguration;");
Body.pushln("import org.apache.giraph.GiraphConfiguration;");
Body.pushln("import org.apache.giraph.graph.GiraphJob;");
Body.pushln("import org.apache.giraph.io.formats.GiraphFileInputFormat;");
Body.pushln("import org.apache.giraph.io.GiraphFileInputFormat;");
Body.pushln("import org.apache.hadoop.conf.Configuration;");
Body.pushln("import org.apache.hadoop.fs.Path;");
Body.pushln("import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;");
Expand All @@ -64,9 +62,11 @@ void gm_giraphlib::generate_headers_input(gm_code_writer& Body) {
generate_package_decl_if_required(Body);
Body.pushln("import java.io.IOException;");
Body.pushln("import java.util.List;");
Body.pushln("import java.util.Map;");
Body.pushln("import java.util.HashMap;");

Body.pushln("import org.apache.giraph.graph.*;");
Body.pushln("import org.apache.giraph.io.formats.*;");
Body.pushln("import org.apache.giraph.io.*;");
Body.pushln("import org.apache.hadoop.io.*;");
Body.pushln("import org.apache.hadoop.mapreduce.InputSplit;");
Body.pushln("import org.apache.hadoop.mapreduce.TaskAttemptContext;");
Expand All @@ -79,8 +79,7 @@ void gm_giraphlib::generate_headers_output(gm_code_writer& Body) {
Body.pushln("import java.io.IOException;");

Body.pushln("import org.apache.giraph.graph.*;");
Body.pushln("import org.apache.giraph.vertex.*;");
Body.pushln("import org.apache.giraph.io.formats.*;");
Body.pushln("import org.apache.giraph.io.*;");
Body.pushln("import org.apache.hadoop.io.*;");
Body.pushln("import org.apache.hadoop.mapreduce.TaskAttemptContext;");
Body.NL();
Expand Down Expand Up @@ -471,12 +470,12 @@ static void genPutIOBCollection(const char* name, ast_typedecl* T, gm_code_write
Body.pushln(") {");
if (((base_type == GMTYPE_NODE) && (lib->is_node_type_int())) ||
((base_type == GMTYPE_EDGE) && (lib->is_edge_type_int()))) {
Body.push("out.WriteInt(");
Body.push("out.writeInt(");
Body.push(iterator);
Body.pushln(".get());");
}
else {
Body.push("out.WriteLong(");
Body.push("out.writeLong(");
Body.push(iterator);
Body.pushln(".get());");
}
Expand Down Expand Up @@ -531,8 +530,8 @@ static void genGetIOBCollection(const char* name, ast_typedecl* T, gm_code_write
int base_type = T->is_node_collection() ? GMTYPE_NODE : GMTYPE_EDGE;
const char* iterator = "__iter";
Body.pushln("{");
Body.pushln("int __size = out.readInt();");
Body.pushln("for (__i = 0; __i < __size; __i++) {");
Body.pushln("int __size = in.readInt();");
Body.pushln("for (int __i = 0; __i < __size; __i++) {");
Body.push(name);
Body.push(".add(new ");
Body.push(lib->get_main()->get_box_type_string(base_type));
Expand Down
5 changes: 3 additions & 2 deletions src/backend_gps/gm_gps_new_check_nested_loops.cc
View file
Expand Up @@ -28,10 +28,11 @@
// GPS_FLAG_IS_OUTER_LOOP: <to:>foreach or symbol of iterator <what:>true if outer loop
//------------------------------------------------------------------------


class gps_new_check_nested_if_used_as_driver_t : public gm_apply
{
public:
gps_new_check_nested_if_used_as_driver_t(std::set<gm_symtab_entry*>& _S) : S(_S) {
gps_new_check_nested_if_used_as_driver_t(std::set<gm_symtab_entry*>& __S) : S(__S) {
set_for_builtin(true);
set_for_lhs(true);
set_for_rhs(true);
Expand Down Expand Up @@ -83,7 +84,7 @@ class gps_new_check_nested_if_used_as_driver_t : public gm_apply
class gps_new_check_nested_depth_t : public gm_apply
{
public:
gps_new_check_nested_depth_t(std::set<gm_symtab_entry*>& _S, bool b) : S(_S), _gen_error(b) {
gps_new_check_nested_depth_t(std::set<gm_symtab_entry*>& __S, bool b) : S(__S), _gen_error(b) {
_error = false;
inner_loop = NULL;
outer_loop = NULL;
Expand Down
15 changes: 13 additions & 2 deletions test/gps/Makefile
View file
@@ -1,5 +1,6 @@

# stop after parse and typecheck
include ../../setup.mk

ODIR = output
COMP = ../../bin/gm_comp
FLAGS = -o=$(ODIR) -t=giraph
Expand All @@ -18,4 +19,14 @@ $(ODIR)/result_%.gm: %.gm
$(COMP) $(FLAGS) -DS=5 -V=1 $< > $@

clean:
\rm -rf $(ODIR)/*.java *.java $(ODIR)/*.gm *.cc *.h
\rm -rf $(ODIR)/*.java *.java $(ODIR)/*.gm *.cc *.h $(ODIR)/*.jar

GIRAPH_JAR=giraph-0.2-SNAPSHOT-for-hadoop-0.20.203.0-jar-with-dependencies.jar
TEST_JAR = test.jar
jar: output/$(TEST_JAR)

output/$(TEST_JAR):
mkdir -p lib
mkdir -p bin
javac output/*.java -Xlint -d bin/ -classpath $(GM_TOP)/apps/output_giraph/lib/$(GIRAPH_JAR):$(HADOOP_HOME)/$(HADOOP_CORE_JAR)
jar cf output/$(TEST_JAR) -C bin/ .

0 comments on commit e04f700

Please sign in to comment.