/
ZippedDocumentAppender.java
159 lines (119 loc) · 4.55 KB
/
ZippedDocumentAppender.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
package net.openhft.chronicle.queue.impl.ringbuffer;
import net.openhft.chronicle.bytes.Bytes;
import net.openhft.chronicle.queue.impl.DirectChronicleQueue;
import net.openhft.chronicle.threads.NamedThreadFactory;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.ExecutorService;
import java.util.zip.Deflater;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
/**
* Create a background thread to zip the {@code bytes} before appending the zipped bytes to the
* ChronicleQueue, in the meantime the bytes are held in a ring buffer.
*
* @author Rob Austin.
*/
public class ZippedDocumentAppender implements Closeable {
private static final Logger LOG = LoggerFactory.getLogger(ZippedDocumentAppender.class.getName());
@NotNull
private final BytesRingBuffer q;
@NotNull
private final DirectChronicleQueue chronicleQueue;
public static final String ERR_MSG = "It not possible to zip more than " +
"Integer.MAX_VALUE bytes in one go";
@NotNull
private final ExecutorService qReader;
/**
* @param bytesRingBuffer a ring buffer to hold the bytes before they are zipped
* @param chronicleQueue the chronicle you wish to append the zipped bytes to
*/
public ZippedDocumentAppender(@NotNull final BytesRingBuffer bytesRingBuffer,
@NotNull final DirectChronicleQueue chronicleQueue) {
this.q = bytesRingBuffer;
this.chronicleQueue = chronicleQueue;
qReader = newSingleThreadExecutor(new NamedThreadFactory("qReader"));
qReader.submit(new Consumer());
}
/**
* the bytes that you wish to append, this bytes will become zipped and appended to the
* chronicle using a background thread
*
* @param bytes the bytes to append
* @throws InterruptedException
*/
public void append(@NotNull Bytes bytes) throws InterruptedException {
//noinspection StatementWithEmptyBody
while (!q.offer(bytes)) {
}
}
/**
* terminates the background thread
*
* @throws IOException
*/
@Override
public void close() throws IOException {
qReader.shutdown();
}
/**
* used to consumer bytes out of the ring buffer, zip up the bytes using the {@code compresser} and write these
* zipped bytes to {@code chronicleQueue}
*/
private class Consumer implements BytesRingBuffer.BytesProvider, Runnable {
@NotNull
private Deflater compresser = new Deflater();
@NotNull
private byte[] input = new byte[]{};
@NotNull
private byte[] output = new byte[]{};
@NotNull
private Bytes<ByteBuffer> inputBuffer = Bytes.wrap(ByteBuffer.wrap(input));
@NotNull
private Bytes<ByteBuffer> outputBuffer = Bytes.wrap(ByteBuffer.wrap(input));
private Consumer() {
this.input = new byte[]{};
this.inputBuffer = Bytes.wrap(ByteBuffer.wrap(input));
}
@Override
public void run() {
try {
for (; ; ) {
if (Thread.currentThread().isInterrupted())
return;
Bytes value;
do {
value = q.poll(this);
} while (value == null);
compresser.setInput(input, (int) value.position(), (int) value.remaining());
compresser.finish();
int limit = compresser.deflate(output);
compresser.end();
outputBuffer.position(0);
outputBuffer.limit(limit);
chronicleQueue.appendDocument(outputBuffer);
}
} catch (Exception e) {
LOG.error("", e);
}
}
@NotNull
@Override
public Bytes provide(final long maxSize) {
if (maxSize < inputBuffer.capacity())
return inputBuffer.clear();
if (maxSize > Integer.MAX_VALUE) {
throw new IllegalStateException(ERR_MSG);
}
// resize the buffers
this.input = new byte[(int) maxSize];
this.inputBuffer = Bytes.wrap(ByteBuffer.wrap(input));
this.output = new byte[(int) maxSize];
this.outputBuffer = Bytes.wrap(ByteBuffer.wrap(output));
return inputBuffer;
}
}
}