|
@@ -19,15 +19,12 @@ |
|
|
import java.util.Collections; |
|
|
import java.util.Iterator; |
|
|
import java.util.concurrent.Callable; |
|
|
import java.util.concurrent.CompletableFuture; |
|
|
import java.util.concurrent.ExecutorService; |
|
|
import java.util.concurrent.TimeUnit; |
|
|
|
|
|
import com.google.common.collect.ImmutableList; |
|
|
import com.google.common.collect.ImmutableList.Builder; |
|
|
import com.google.common.util.concurrent.AsyncFunction; |
|
|
import com.google.common.util.concurrent.Futures; |
|
|
import com.google.common.util.concurrent.ListenableFuture; |
|
|
import com.google.common.util.concurrent.ListeningExecutorService; |
|
|
import com.google.common.util.concurrent.MoreExecutors; |
|
|
import org.apache.accumulo.core.client.ConditionalWriter; |
|
|
import org.apache.accumulo.core.client.ConditionalWriter.Result; |
|
|
import org.apache.accumulo.core.data.ConditionalMutation; |
|
@@ -36,11 +33,10 @@ |
|
|
import org.apache.fluo.core.util.FluoExecutors; |
|
|
import org.apache.fluo.core.util.Limit; |
|
|
|
|
|
public class AsyncConditionalWriter |
|
|
implements AsyncFunction<Collection<ConditionalMutation>, Iterator<Result>> { |
|
|
public class AsyncConditionalWriter { |
|
|
|
|
|
private final ConditionalWriter cw; |
|
|
private final ListeningExecutorService les; |
|
|
private final ExecutorService es; |
|
|
private final Limit semaphore; |
|
|
|
|
|
|
|
@@ -50,55 +46,38 @@ public AsyncConditionalWriter(Environment env, ConditionalWriter cw) { |
|
|
FluoConfigurationImpl.ASYNC_CW_THREADS_DEFAULT); |
|
|
int permits = env.getConfiguration().getInt(FluoConfigurationImpl.ASYNC_CW_LIMIT, |
|
|
FluoConfigurationImpl.ASYNC_CW_LIMIT_DEFAULT); |
|
|
this.les = |
|
|
MoreExecutors.listeningDecorator(FluoExecutors.newFixedThreadPool(numThreads, "asyncCW")); |
|
|
this.es = FluoExecutors.newFixedThreadPool(numThreads, "asyncCw"); |
|
|
// the conditional writer currently has not memory limits... give it too much and it blows out |
|
|
// memory.. need to fix this in conditional writer |
|
|
// for now this needs to be memory based |
|
|
this.semaphore = new Limit(permits); |
|
|
} |
|
|
|
|
|
private class IterTask implements Callable<Iterator<Result>> { |
|
|
|
|
|
private Iterator<Result> input; |
|
|
private int permitsAcquired; |
|
|
|
|
|
public IterTask(Iterator<Result> iter, int permitsAcquired) { |
|
|
this.input = iter; |
|
|
this.permitsAcquired = permitsAcquired; |
|
|
public CompletableFuture<Iterator<Result>> apply(Collection<ConditionalMutation> input) { |
|
|
if (input.size() == 0) { |
|
|
return CompletableFuture.completedFuture(Collections.<Result>emptyList().iterator()); |
|
|
} |
|
|
|
|
|
@Override |
|
|
public Iterator<Result> call() throws Exception { |
|
|
semaphore.acquire(input.size()); |
|
|
Iterator<Result> iter = cw.write(input.iterator()); |
|
|
return CompletableFuture.supplyAsync(() -> { |
|
|
try { |
|
|
Builder<Result> imlb = ImmutableList.builder(); |
|
|
while (input.hasNext()) { |
|
|
Result result = input.next(); |
|
|
while (iter.hasNext()) { |
|
|
Result result = iter.next(); |
|
|
imlb.add(result); |
|
|
} |
|
|
return imlb.build().iterator(); |
|
|
} finally { |
|
|
semaphore.release(permitsAcquired); |
|
|
semaphore.release(input.size()); |
|
|
} |
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
@Override |
|
|
public ListenableFuture<Iterator<Result>> apply(Collection<ConditionalMutation> input) { |
|
|
if (input.size() == 0) { |
|
|
return Futures.immediateFuture(Collections.<Result>emptyList().iterator()); |
|
|
} |
|
|
|
|
|
semaphore.acquire(input.size()); |
|
|
Iterator<Result> iter = cw.write(input.iterator()); |
|
|
return les.submit(new IterTask(iter, input.size())); |
|
|
}, es); |
|
|
} |
|
|
|
|
|
public void close() { |
|
|
les.shutdownNow(); |
|
|
es.shutdownNow(); |
|
|
try { |
|
|
les.awaitTermination(5, TimeUnit.SECONDS); |
|
|
es.awaitTermination(5, TimeUnit.SECONDS); |
|
|
} catch (InterruptedException e) { |
|
|
throw new RuntimeException(e); |
|
|
} |
|
|