Skip to content
Permalink
Browse files
Merge pull request #77 from mikewalch/apache-rename
Updated package naming for Apache Fluo
  • Loading branch information
keith-turner committed Jun 17, 2016
2 parents e471af0 + a156f00 commit 78ce3b456330020c964f971796542aab6cf93e01
Show file tree
Hide file tree
Showing 23 changed files with 187 additions and 188 deletions.
@@ -16,7 +16,6 @@

import java.text.ParseException;

import io.fluo.webindex.core.models.URL;
import org.junit.Assert;
import org.junit.Test;

@@ -45,46 +45,46 @@
</dependency>
<dependency>
<groupId>io.fluo</groupId>
<artifactId>webindex-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.fluo</groupId>
<artifactId>fluo-api</artifactId>
</dependency>
<dependency>
<groupId>io.fluo</groupId>
<groupId>org.apache.fluo</groupId>
<artifactId>fluo-core</artifactId>
</dependency>
<dependency>
<groupId>io.fluo</groupId>
<groupId>org.apache.fluo</groupId>
<artifactId>fluo-mapreduce</artifactId>
</dependency>
<dependency>
<groupId>io.fluo</groupId>
<groupId>org.apache.fluo</groupId>
<artifactId>fluo-recipes-accumulo</artifactId>
</dependency>
<dependency>
<groupId>io.fluo</groupId>
<groupId>org.apache.fluo</groupId>
<artifactId>fluo-recipes-core</artifactId>
</dependency>
<dependency>
<groupId>io.fluo</groupId>
<groupId>org.apache.fluo</groupId>
<artifactId>fluo-recipes-kryo</artifactId>
</dependency>
<dependency>
<groupId>io.fluo</groupId>
<groupId>org.apache.fluo</groupId>
<artifactId>fluo-recipes-spark</artifactId>
</dependency>
<dependency>
<groupId>io.fluo</groupId>
<artifactId>webindex-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -125,8 +125,14 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>io.fluo</groupId>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.fluo</groupId>
<artifactId>fluo-mini</artifactId>
<scope>test</scope>
<exclusions>
@@ -137,16 +143,10 @@
</exclusions>
</dependency>
<dependency>
<groupId>io.fluo</groupId>
<groupId>org.apache.fluo</groupId>
<artifactId>fluo-recipes-test</artifactId>
<scope>test</scope>
</dependency>
<!-- Test Dependencies -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
@@ -16,14 +16,14 @@

import java.util.SortedSet;

import io.fluo.api.data.Bytes;
import io.fluo.api.data.RowColumn;
import io.fluo.webindex.core.models.Page;
import io.fluo.webindex.data.fluo.UriMap.UriInfo;
import io.fluo.webindex.data.spark.IndexEnv;
import io.fluo.webindex.data.spark.IndexStats;
import io.fluo.webindex.data.spark.IndexUtil;
import io.fluo.webindex.data.util.WARCFileInputFormat;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.RowColumn;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.spark.SparkConf;
@@ -19,9 +19,9 @@
import java.io.PrintWriter;
import java.util.Iterator;

import io.fluo.api.config.FluoConfiguration;
import io.fluo.webindex.core.DataConfig;
import io.fluo.webindex.data.spark.IndexEnv;
import org.apache.fluo.api.config.FluoConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@@ -14,17 +14,17 @@

package io.fluo.webindex.data;

import io.fluo.api.config.FluoConfiguration;
import io.fluo.api.config.ObserverConfiguration;
import io.fluo.recipes.accumulo.export.AccumuloExport;
import io.fluo.recipes.accumulo.export.AccumuloExporter;
import io.fluo.recipes.accumulo.export.TableInfo;
import io.fluo.recipes.export.ExportQueue;
import io.fluo.recipes.kryo.KryoSimplerSerializer;
import io.fluo.webindex.data.fluo.DomainMap;
import io.fluo.webindex.data.fluo.PageObserver;
import io.fluo.webindex.data.fluo.UriMap;
import io.fluo.webindex.serialization.WebindexKryoFactory;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.config.ObserverConfiguration;
import org.apache.fluo.recipes.accumulo.export.AccumuloExport;
import org.apache.fluo.recipes.accumulo.export.AccumuloExporter;
import org.apache.fluo.recipes.accumulo.export.TableInfo;
import org.apache.fluo.recipes.export.ExportQueue;
import org.apache.fluo.recipes.kryo.KryoSimplerSerializer;

public class FluoApp {

@@ -20,15 +20,15 @@
import java.util.List;

import com.google.common.util.concurrent.RateLimiter;
import io.fluo.api.client.FluoClient;
import io.fluo.api.client.FluoFactory;
import io.fluo.api.client.LoaderExecutor;
import io.fluo.api.config.FluoConfiguration;
import io.fluo.webindex.core.DataConfig;
import io.fluo.webindex.core.models.Page;
import io.fluo.webindex.data.fluo.PageLoader;
import io.fluo.webindex.data.spark.IndexEnv;
import io.fluo.webindex.data.util.ArchiveUtil;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
import org.apache.fluo.api.client.LoaderExecutor;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -19,15 +19,15 @@
import java.util.List;

import com.google.common.util.concurrent.RateLimiter;
import io.fluo.api.client.FluoClient;
import io.fluo.api.client.FluoFactory;
import io.fluo.api.client.LoaderExecutor;
import io.fluo.api.config.FluoConfiguration;
import io.fluo.webindex.core.DataConfig;
import io.fluo.webindex.core.models.Page;
import io.fluo.webindex.data.fluo.PageLoader;
import io.fluo.webindex.data.spark.IndexEnv;
import io.fluo.webindex.data.util.ArchiveUtil;
import org.apache.fluo.api.client.FluoClient;
import org.apache.fluo.api.client.FluoFactory;
import org.apache.fluo.api.client.LoaderExecutor;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
@@ -18,9 +18,9 @@
import java.util.Map;
import java.util.Optional;

import io.fluo.api.data.Bytes;
import io.fluo.api.data.RowColumn;
import io.fluo.recipes.accumulo.export.DifferenceExport;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.RowColumn;
import org.apache.fluo.recipes.accumulo.export.DifferenceExport;

import static io.fluo.webindex.data.util.FluoConstants.PAGECOUNT_COL;

@@ -17,17 +17,17 @@
import java.util.Iterator;
import java.util.Optional;

import io.fluo.api.client.TransactionBase;
import io.fluo.api.config.FluoConfiguration;
import io.fluo.api.observer.Observer.Context;
import io.fluo.recipes.accumulo.export.AccumuloExport;
import io.fluo.recipes.export.ExportQueue;
import io.fluo.recipes.map.CollisionFreeMap;
import io.fluo.recipes.map.CollisionFreeMap.Options;
import io.fluo.recipes.map.Combiner;
import io.fluo.recipes.map.Update;
import io.fluo.recipes.map.UpdateObserver;
import io.fluo.webindex.data.FluoApp;
import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.observer.Observer.Context;
import org.apache.fluo.recipes.accumulo.export.AccumuloExport;
import org.apache.fluo.recipes.export.ExportQueue;
import org.apache.fluo.recipes.map.CollisionFreeMap;
import org.apache.fluo.recipes.map.CollisionFreeMap.Options;
import org.apache.fluo.recipes.map.Combiner;
import org.apache.fluo.recipes.map.Update;
import org.apache.fluo.recipes.map.UpdateObserver;

public class DomainMap {
public static final String DOMAIN_MAP_ID = "dm";
@@ -19,11 +19,11 @@
import java.util.List;

import com.google.common.collect.Sets.SetView;
import io.fluo.recipes.accumulo.export.AccumuloExport;
import io.fluo.webindex.core.Constants;
import io.fluo.webindex.core.models.Link;
import io.fluo.webindex.core.models.Page;
import org.apache.accumulo.core.data.Mutation;
import org.apache.fluo.recipes.accumulo.export.AccumuloExport;

public class PageExport implements AccumuloExport<String> {

@@ -19,13 +19,13 @@

import com.google.common.base.Preconditions;
import com.google.gson.Gson;
import io.fluo.api.client.Loader;
import io.fluo.api.client.TransactionBase;
import io.fluo.api.types.TypedTransactionBase;
import io.fluo.recipes.data.RowHasher;
import io.fluo.webindex.core.models.Page;
import io.fluo.webindex.core.models.URL;
import io.fluo.webindex.data.util.FluoConstants;
import org.apache.fluo.api.client.Loader;
import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.api.types.TypedTransactionBase;
import org.apache.fluo.recipes.data.RowHasher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@@ -20,21 +20,21 @@

import com.google.common.collect.Sets;
import com.google.gson.Gson;
import io.fluo.api.client.TransactionBase;
import io.fluo.api.data.Bytes;
import io.fluo.api.data.Column;
import io.fluo.api.observer.AbstractObserver;
import io.fluo.api.types.TypedSnapshotBase.Value;
import io.fluo.api.types.TypedTransactionBase;
import io.fluo.recipes.accumulo.export.AccumuloExport;
import io.fluo.recipes.data.RowHasher;
import io.fluo.recipes.export.ExportQueue;
import io.fluo.recipes.map.CollisionFreeMap;
import io.fluo.webindex.core.models.Link;
import io.fluo.webindex.core.models.Page;
import io.fluo.webindex.data.FluoApp;
import io.fluo.webindex.data.fluo.UriMap.UriInfo;
import io.fluo.webindex.data.util.FluoConstants;
import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.Column;
import org.apache.fluo.api.observer.AbstractObserver;
import org.apache.fluo.api.types.TypedSnapshotBase.Value;
import org.apache.fluo.api.types.TypedTransactionBase;
import org.apache.fluo.recipes.accumulo.export.AccumuloExport;
import org.apache.fluo.recipes.data.RowHasher;
import org.apache.fluo.recipes.export.ExportQueue;
import org.apache.fluo.recipes.map.CollisionFreeMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

@@ -19,10 +19,6 @@
import java.util.Map;
import java.util.Optional;

import io.fluo.api.data.Bytes;
import io.fluo.api.data.Column;
import io.fluo.api.data.RowColumn;
import io.fluo.recipes.accumulo.export.DifferenceExport;
import io.fluo.webindex.core.Constants;
import io.fluo.webindex.core.models.URL;
import io.fluo.webindex.data.fluo.UriMap.UriInfo;
@@ -31,6 +27,10 @@
import org.apache.accumulo.core.client.lexicoder.ReverseLexicoder;
import org.apache.accumulo.core.client.lexicoder.ULongLexicoder;
import org.apache.commons.codec.binary.Hex;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.Column;
import org.apache.fluo.api.data.RowColumn;
import org.apache.fluo.recipes.accumulo.export.DifferenceExport;

public class UriCountExport extends DifferenceExport<String, UriInfo> {

@@ -21,18 +21,18 @@
import java.util.Optional;

import com.google.common.base.Preconditions;
import io.fluo.api.client.TransactionBase;
import io.fluo.api.config.FluoConfiguration;
import io.fluo.api.observer.Observer.Context;
import io.fluo.recipes.accumulo.export.AccumuloExport;
import io.fluo.recipes.export.ExportQueue;
import io.fluo.recipes.map.CollisionFreeMap;
import io.fluo.recipes.map.CollisionFreeMap.Options;
import io.fluo.recipes.map.Combiner;
import io.fluo.recipes.map.Update;
import io.fluo.recipes.map.UpdateObserver;
import io.fluo.webindex.core.models.URL;
import io.fluo.webindex.data.FluoApp;
import org.apache.fluo.api.client.TransactionBase;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.observer.Observer.Context;
import org.apache.fluo.recipes.accumulo.export.AccumuloExport;
import org.apache.fluo.recipes.export.ExportQueue;
import org.apache.fluo.recipes.map.CollisionFreeMap;
import org.apache.fluo.recipes.map.CollisionFreeMap.Options;
import org.apache.fluo.recipes.map.Combiner;
import org.apache.fluo.recipes.map.Update;
import org.apache.fluo.recipes.map.UpdateObserver;

/**
* This class contains code related to a CollisionFreeMap that keeps track of the count of
@@ -29,15 +29,6 @@
import java.util.stream.Stream;

import com.google.common.base.Preconditions;
import io.fluo.api.config.FluoConfiguration;
import io.fluo.api.data.Bytes;
import io.fluo.api.data.RowColumn;
import io.fluo.core.util.AccumuloUtil;
import io.fluo.recipes.accumulo.export.TableInfo;
import io.fluo.recipes.accumulo.ops.TableOperations;
import io.fluo.recipes.common.Pirtos;
import io.fluo.recipes.spark.FluoSparkHelper;
import io.fluo.recipes.spark.FluoSparkHelper.BulkImportOptions;
import io.fluo.webindex.core.DataConfig;
import io.fluo.webindex.core.models.Page;
import io.fluo.webindex.data.FluoApp;
@@ -48,6 +39,15 @@
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.fluo.api.config.FluoConfiguration;
import org.apache.fluo.api.data.Bytes;
import org.apache.fluo.api.data.RowColumn;
import org.apache.fluo.core.util.AccumuloUtil;
import org.apache.fluo.recipes.accumulo.export.TableInfo;
import org.apache.fluo.recipes.accumulo.ops.TableOperations;
import org.apache.fluo.recipes.common.Pirtos;
import org.apache.fluo.recipes.spark.FluoSparkHelper;
import org.apache.fluo.recipes.spark.FluoSparkHelper.BulkImportOptions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;

0 comments on commit 78ce3b4

Please sign in to comment.