Too many files Issue with OKHttp #805
Description
In the following test, we are adding documents using WriteBatcher, with multiple Threads. The ulimit value on the server has been set to 20,000 for both soft and hard limits. We see that the file descriptors are not getting released after document writes and they increase upto the ulimit value set on the server. The test eventually fails to write additional documents and throws the following exception.
This test ran successfully with 4.0.1 jar file on the same server, with same ulimit values.
15:28:30.027 [pool-133-thread-1] ERROR c.m.c.d.HostAvailabilityListener - Encountered [com.marklogic.client.MarkLogicIOException: java.net.SocketException: Too many open files] on host "localhost" but black-listing it would drop job below minHosts (1), so stopping job "unnamed"
com.marklogic.client.MarkLogicIOException: java.net.SocketException: Too many open files
at com.marklogic.client.impl.OkHttpServices.sendRequestOnce(OkHttpServices.java:675) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at com.marklogic.client.impl.OkHttpServices.sendRequestOnce(OkHttpServices.java:668) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at com.marklogic.client.impl.OkHttpServices.doPost(OkHttpServices.java:3991) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at com.marklogic.client.impl.OkHttpServices.postResource(OkHttpServices.java:3292) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at com.marklogic.client.impl.OkHttpServices.postBulkDocuments(OkHttpServices.java:3400) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at com.marklogic.client.impl.DocumentManagerImpl.write(DocumentManagerImpl.java:613) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at com.marklogic.client.impl.GenericDocumentImpl.write(GenericDocumentImpl.java:23) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at com.marklogic.client.impl.DocumentManagerImpl.write(DocumentManagerImpl.java:605) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at com.marklogic.client.impl.GenericDocumentImpl.write(GenericDocumentImpl.java:23) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at com.marklogic.client.datamovement.impl.WriteBatcherImpl$BatchWriter.run(WriteBatcherImpl.java:1020) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[na:1.8.0_72]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_72]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) ~[na:1.8.0_72]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) ~[na:1.8.0_72]
at java.lang.Thread.run(Thread.java:745) ~[na:1.8.0_72]
Caused by: java.net.SocketException: Too many open files
at java.net.Socket.createImpl(Socket.java:460) ~[na:1.8.0_72]
at java.net.Socket.getImpl(Socket.java:520) ~[na:1.8.0_72]
at java.net.Socket.setSoTimeout(Socket.java:1141) ~[na:1.8.0_72]
at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.java:221) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.connection.RealConnection.connect(RealConnection.java:149) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:195) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:121) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:100) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.8.1.jar:na]
at com.burgstaller.okhttp.AuthenticationCacheInterceptor.intercept(AuthenticationCacheInterceptor.java:45) ~[okhttp-digest-1.14.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) ~[okhttp-3.8.1.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) ~[okhttp-3.8.1.jar:na]
at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:185) ~[okhttp-3.8.1.jar:na]
at okhttp3.RealCall.execute(RealCall.java:69) ~[okhttp-3.8.1.jar:na]
at com.marklogic.client.impl.OkHttpServices.sendRequestOnce(OkHttpServices.java:673) ~[marklogic-client-api-5.0-SNAPSHOT.jar:na]
... 14 common frames omitted
@Test
public void testAddFileOpenIssue() throws Exception {
final String query1 = "fn:count(fn:doc())";
ihbMT = dmManager.newWriteBatcher();
ihbMT.withBatchSize(99);
ihbMT.withThreadCount(10);
ihbMT.onBatchSuccess(
batch -> {
}
)
.onBatchFailure(
(batch, throwable) -> {
throwable.printStackTrace();
});
dmManager.startJob(ihbMT);
class MyRunnable implements Runnable {
@Override
public void run() {
for (int j = 0; j < 50000; j++) {
String uri = "/local/json-" + j + "-" + Thread.currentThread().getId();
System.out.println("Thread name: " + Thread.currentThread().getName() + " URI:" + uri);
ihbMT.add(uri, fileHandle);
}
ihbMT.flushAndWait();
}
}
Thread t1, t2, t3;
t1 = new Thread(new MyRunnable());
t2 = new Thread(new MyRunnable());
t3 = new Thread(new MyRunnable());
t1.start();
t2.start();
t3.start();
t1.join();
t2.join();
t3.join();
Assert.assertTrue(dbClient.newServerEval().xquery(query1).eval().next().getNumber().intValue() == 150000);
}