Permalink
Browse files

HADOOP-18: Add Pig build, Remove Maven POMs; also culled some extra e…

…xamples for the time being
  • Loading branch information...
1 parent 0bc79d7 commit 44bb93b8944e3157f945b49ce741a4aecbc6fad3 @bwmcadams bwmcadams committed Feb 12, 2012
Showing with 5 additions and 2,107 deletions.
  1. +0 −146 core/pom.xml
  2. +0 −44 examples/snmp/config/Config-Snmp.xml
  3. +0 −44 examples/snmp/config/ConfigSnmpChain.xml
  4. +0 −44 examples/snmp/config/ConfigSnmpWithCombiner.xml
  5. +0 −14 examples/snmp/config/README
  6. +0 −170 examples/snmp/config/build.xml
  7. +0 −51 examples/snmp/config/buildScript.sh
  8. +0 −7 examples/snmp/config/importdb.sh
  9. +0 −11 examples/snmp/config/setClassPath.sh
  10. +0 −20 examples/snmp/config/splitBigFile.sh
  11. +0 −22 examples/snmp/pom.xml
  12. +0 −134 examples/snmp/src/main/java/com/mongodb/hadoop/examples/snmp/SnmpStatisticMongoTool.java
  13. +0 −160 examples/snmp/src/main/java/com/mongodb/hadoop/examples/snmp/SnmpStatisticWithCombiner.java
  14. +0 −209 examples/snmp/src/main/java/com/mongodb/hadoop/examples/snmp/SnmpStatistic_MapReduceChain.java
  15. +0 −4 examples/world_development_indicators/README
  16. +0 −22 examples/world_development_indicators/pom.xml
  17. +0 −139 ...tors/src/main/java/com/mongodb/hadoop/examples/world_development/WorldDevIndicatorDataLoader.java
  18. +0 −97 ...dicators/src/main/java/com/mongodb/hadoop/examples/world_development/WorldDevIndicatorMapper.java
  19. +0 −55 ...icators/src/main/java/com/mongodb/hadoop/examples/world_development/WorldDevIndicatorReducer.java
  20. +0 −41 ...ators/src/main/java/com/mongodb/hadoop/examples/world_development/WorldDevIndicatorXmlConfig.java
  21. BIN examples/world_development_indicators/src/main/resources/WDI_GDF_Data.csv.zip
  22. +0 −40 examples/world_development_indicators/src/main/resources/hadoop-local.xml
  23. +0 −159 examples/world_development_indicators/src/main/resources/mongo-defaults.xml
  24. +0 −137 examples/world_development_indicators/src/main/resources/world-development-indicators.xml
  25. +0 −42 flume/pom.xml
  26. +0 −49 pig/pom.xml
  27. +0 −176 pom.xml
  28. +5 −0 project/MongoHadoopBuild.scala
  29. +0 −70 streaming/pom.xml
View
@@ -1,146 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-hadoop-core</artifactId>
- <packaging>jar</packaging>
- <version>1.0-SNAPSHOT</version>
- <name>MongoDB Hadoop Core Support</name>
- <url>http://github.com/mongodb/mongo-hadoop</url>
- <properties>
- <scala.version>2.9.1</scala.version>
- </properties>
- <parent>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-hadoop</artifactId>
- <version>1.0-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
- <repositories>
- <repository>
- <id>scala-tools.org</id>
- <name>Scala-tools Maven2 Repository</name>
- <url>http://scala-tools.org/repo-releases</url>
- </repository>
- </repositories>
- <pluginRepositories>
- <pluginRepository>
- <id>scala-tools.org</id>
- <name>Scala-tools Maven2 Repository</name>
- <url>http://scala-tools.org/repo-releases</url>
- </pluginRepository>
- </pluginRepositories>
- <dependencies>
- <dependency>
- <groupId>org.mongodb</groupId>
- <artifactId>mongo-java-driver</artifactId>
- <version>2.7.3</version>
- </dependency>
-
- <dependency>
- <groupId>org.specs2</groupId>
- <artifactId>specs2_${scala.version}</artifactId>
- <version>1.7.1</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>${hadoop.baseVersion}</version>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.8.1</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- <filtering>true</filtering>
- </resource>
- </resources>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <version>${scala.version}</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.0.2</version>
- </plugin>
- </plugins>
- </pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <executions>
- <execution>
- <id>scala-compile-first</id>
- <phase>process-resources</phase>
- <goals>
- <goal>add-source</goal>
- <goal>compile</goal>
- </goals>
- </execution>
- <execution>
- <id>scala-test-compile</id>
- <phase>process-test-resources</phase>
- <goals>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <scalaVersion>${scala.version}</scalaVersion>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.5</source>
- <target>1.5</target>
- </configuration>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
-
- </plugin>
- <plugin>
- <groupId>com.mmakowski</groupId>
- <artifactId>maven-specs2-plugin</artifactId>
- <version>0.2.1</version>
- <executions>
- <execution>
- <id>test</id>
- <phase>test</phase>
- <goals>
- <goal>run-specs</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
@@ -1,44 +0,0 @@
-<?xml version="1.0"?>
-<configuration>
-<testparams>
-
-
-<property name="mongo.input.split.read_shard_chunks">
- <val value="true"/>
- <val value="false" baseline="true"/>
-</property>
-
-<property name="mongo.input.split.read_from_shards">
- <val value="true" />
- <val value="false" baseline="true"/>
-</property>
-
-<property name="mongo.input.split.allow_read_from_secondaries">
- <when name="mongo.input.split.read_from_shards" val="true"/>
- <val value="true"/>
- <val value="false"/>
-
- </property>
-
-<!--property>
- <name>mapred.job.tracker</name>
- <value>ip-10-203-3-139.ec2.internal:8021</value>
-</property-->
-
-</testparams>
-
-<testcases>
-<test class="SnmpStatisticMongoTool" args="">
-<property name="mongo.input.uri" val="mongodb://10.202.149.73:27018/test.snmp"></property>
-<property name="mongo.output.uri" val="mongodb://10.202.149.73:27018/test.temp_out"></property>
-<property name="mapred.job.tracker" val="ip-10-203-3-139.ec2.internal:8021"></property>
-</test>
-
-</testcases>
-
-<resultsdb uri="mongodb://localhost/results.output" />
-<path binpath="" dumppath="" />
-<!-- old: dbprops dbname="test" dbout="results" port="30000" binpath="/home/rushin/mongodb-src/"></dbprops -->
-</configuration>
-
-
@@ -1,44 +0,0 @@
-<?xml version="1.0"?>
-<configuration>
-<testparams>
-
-
-<property name="mongo.input.split.read_shard_chunks">
- <val value="true"/>
- <val value="false" baseline="true"/>
-</property>
-
-<property name="mongo.input.split.read_from_shards">
- <val value="true" />
- <val value="false" baseline="true"/>
-</property>
-
-<property name="mongo.input.split.allow_read_from_secondaries">
- <when name="mongo.input.split.read_from_shards" val="true"/>
- <val value="true"/>
- <val value="false"/>
-
- </property>
-
-<!--property>
- <name>mapred.job.tracker</name>
- <value>ip-10-203-3-139.ec2.internal:8021</value>
-</property-->
-
-</testparams>
-
-<testcases>
-<test class="SnmpStatistic_MapReduceChain" args="">
-<property name="mongo.input.uri" val="mongodb://10.202.149.73:27018/test.snmp"></property>
-<property name="mongo.output.uri" val="mongodb://10.202.149.73:27018/test.temp_out_1"></property>
-<property name="mapred.job.tracker" val="ip-10-203-3-139.ec2.internal:8021"></property>
-</test>
-
-</testcases>
-
-<resultsdb uri="mongodb://localhost/results.output" />
-<path binpath="" dumppath="" />
-<!-- old: dbprops dbname="test" dbout="results" port="30000" binpath="/home/rushin/mongodb-src/"></dbprops -->
-</configuration>
-
-
@@ -1,44 +0,0 @@
-<?xml version="1.0"?>
-<configuration>
-<testparams>
-
-
-<property name="mongo.input.split.read_shard_chunks">
- <val value="true"/>
- <val value="false" baseline="true"/>
-</property>
-
-<property name="mongo.input.split.read_from_shards">
- <val value="true" />
- <val value="false" baseline="true"/>
-</property>
-
-<property name="mongo.input.split.allow_read_from_secondaries">
- <when name="mongo.input.split.read_from_shards" val="true"/>
- <val value="true"/>
- <val value="false"/>
-
- </property>
-
-<!--property>
- <name>mapred.job.tracker</name>
- <value>ip-10-203-3-139.ec2.internal:8021</value>
-</property-->
-
-</testparams>
-
-<testcases>
-<test class="SnmpStatisticWithCombiner" args="">
-<property name="mongo.input.uri" val="mongodb://10.202.149.73:27018/test.snmp"></property>
-<property name="mongo.output.uri" val="mongodb://10.202.149.73:27018/test.temp_out_1"></property>
-<property name="mapred.job.tracker" val="ip-10-203-3-139.ec2.internal:8021"></property>
-</test>
-
-</testcases>
-
-<resultsdb uri="mongodb://localhost/results.output" />
-<path binpath="" dumppath="" />
-<!-- old: dbprops dbname="test" dbout="results" port="30000" binpath="/home/rushin/mongodb-src/"></dbprops -->
-</configuration>
-
-
@@ -1,14 +0,0 @@
-1. Download datasets from http://www.crawdad.org/download/dartmouth/campus/snmp/fall03.tar.gz
- Before, you need to register at CRAWDAD for getting username and password.
-
-2. Run buildScript.sh to get script file that can be used to import dataset to MongoDB system.
-
-3. Run splitBigFile.sh because too big script file cannot be imported to MongoDB system.
-
-4. Run importdb.sh to import dataset in form of script file to MongoDB system.
-
-5. Modify original hadoop 2.0 path in build.xml to your hadoop 2.0 path.
-
-6. Compiler MapReduce Funcation by "nice ant"
-
-7. Run "java SnmpStatistic" under "./build/example/snmp/"
Oops, something went wrong. Retry.

0 comments on commit 44bb93b

Please sign in to comment.