-
Notifications
You must be signed in to change notification settings - Fork 23
/
ARCArchiveAccessTester.java
337 lines (292 loc) · 12.8 KB
/
ARCArchiveAccessTester.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
/*
* #%L
* Netarchivesuite - harvester - test
* %%
* Copyright (C) 2005 - 2018 The Royal Danish Library,
* the National Library of France and the Austrian National Library.
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 2.1 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Lesser Public License for more details.
*
* You should have received a copy of the GNU General Lesser Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/lgpl-2.1.html>.
* #L%
*/
package dk.netarkivet.viewerproxy;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Method;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.archive.io.arc .ARCConstants;
import org.archive.io.arc.ARCRecord;
import org.archive.io.arc.ARCRecordMetaData;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import dk.netarkivet.common.distribute.arcrepository.ArcRepositoryClient;
import dk.netarkivet.common.distribute.arcrepository.BitarchiveRecord;
import dk.netarkivet.common.exceptions.ArgumentNotValid;
import dk.netarkivet.common.exceptions.IOFailure;
import dk.netarkivet.common.utils.FileUtils;
import dk.netarkivet.common.utils.Settings;
import dk.netarkivet.common.utils.arc.ARCKey;
import dk.netarkivet.harvester.HarvesterSettings;
import dk.netarkivet.testutils.LogbackRecorder;
import dk.netarkivet.testutils.ReflectUtils;
import dk.netarkivet.testutils.TestFileUtils;
import dk.netarkivet.testutils.preconfigured.ReloadSettings;
/**
* Unit tests for ARCArchiveAccess. This only tests that we connect the CDX lookup with the ARC files ok, because
* everything else is just being forwarded and so is (supposed to be) tested elsewhere.
*/
@SuppressWarnings({"unused", "unchecked"})
public class ARCArchiveAccessTester {
// Unused files:
private static final File MAIN_PATH = new File("tests/dk/netarkivet/viewerproxy/data/");
/**
* An URL not indexed in CDX_FILE. Initiated in setUp because it can throw exception
*/
private static URI WRONG_URL;
/**
* An URL not indexed in CDX_FILE. Initiated in setUp because it can throw exception
*/
private static URI GIF_URL;
/**
* The key listed for GIF_URL.
*/
private static final ARCKey GIF_URL_KEY = new ARCKey(
"2-2-20060731110420-00000-sb-test-har-001.statsbiblioteket.dk.arc", 73269);
// Set up directories for local archive and bitarchive
private static final File BASE_DIR = new File("tests/dk/netarkivet/viewerproxy/data");
private static final File ORIGINALS = new File(BASE_DIR, "input");
private static final File WORKING = new File(BASE_DIR, "working");
// A web archive controller that always returns our own test record:
private ArcRepositoryClient fakeArcRepos;
// A web archive controller that always returns null:
private ArcRepositoryClient nullArcRepos;
// Our main instance of ARCArchiveAccess:
private ARCArchiveAccess aaa;
ReloadSettings rs = new ReloadSettings();
@Before
public void setUp() throws Exception {
rs.setUp();
WRONG_URL = new URI("http://www.test.dk/hest");
GIF_URL = new URI("http://netarkivet.dk/netarchive_alm/billeder/spacer.gif");
// We need a controller that doesn't do much more than return a test
// record:
fakeArcRepos = new TestArcRepositoryClient();
nullArcRepos = new NullArcRepositoryClient();
// Although we also need some real data
FileUtils.removeRecursively(WORKING);
TestFileUtils.copyDirectoryNonCVS(ORIGINALS, WORKING);
Settings.set(HarvesterSettings.VIEWERPROXY_DIR, new File(WORKING, "viewerproxy").getAbsolutePath());
FileUtils.createDir(new File(WORKING, "viewerproxy"));
aaa = new ARCArchiveAccess(fakeArcRepos);
aaa.setIndex(TestInfo.ZIPPED_INDEX_DIR);
}
@After
public void tearDown() {
FileUtils.removeRecursively(WORKING);
rs.tearDown();
}
/**
* Verify that the constructor fails if and only if it is given null parameter.
*/
@Test
public void testConstructor() {
// Verify construction with OK parameters does not fail:
new ARCArchiveAccess(fakeArcRepos);
// Verify that null parameters makes the constructor fail:
try {
new ARCArchiveAccess(null);
fail("Expected argument not valid on null argument.");
} catch (ArgumentNotValid e) {
// Expected.
}
}
@Test
public void testReadPage() throws Exception {
Method readPage = ReflectUtils.getPrivateMethod(ARCArchiveAccess.class, "readPage", InputStream.class,
OutputStream.class);
// Make an inputstream where read(byte[]) reads two chars at a time
InputStream in = new ByteArrayInputStream("foo".getBytes()) {
public int read(byte[] buf) {
return super.read(buf, 0, 2);
}
};
ByteArrayOutputStream baos = new ByteArrayOutputStream();
readPage.invoke(aaa, in, baos);
baos.close();
assertEquals("Should get same bytes regardless of bytes read per read()", "foo", baos.toString());
}
@Test
public void testCreateErrorResponse() throws Exception {
TestResponse response = new TestResponse();
aaa.createNotFoundResponse(WRONG_URL, response);
assertTrue("Url should be in output", response.bas.toString().contains(WRONG_URL.toString()));
assertTrue("Should have content type header", response.names.contains("Content-type"));
assertTrue("Should have test/html header value", response.values.contains("text/html"));
assertEquals("Should be a 404 URL", 404, response.statusInt);
assertEquals("Reason should be as expected", "Not found", response.reasonStr);
response = new TestResponse();
// Test it works twice.
aaa.createNotFoundResponse(WRONG_URL, response);
assertTrue("Url should be in output", response.bas.toString().contains(WRONG_URL.toString()));
assertTrue("Should have content type header", response.names.contains("Content-type"));
assertTrue("Should have test/html header value", response.values.contains("text/html"));
assertEquals("Should be a 404 URL", 404, response.statusInt);
assertEquals("Reason should be as expected", "Not found", response.reasonStr);
}
/**
* Verify that looking up an object that does not exist returns some kind of HTTP response reflecting this
* situation. For now, we just need it to return ResponseCode.NOT_FOUND and not throw an Exception.
*/
@Test
public void testLookupNonexistingObject() throws Exception {
Response response = new TestResponse();
URI uri = new URI("http://does.not.exist");
int code = aaa.lookup(new TestRequest(uri), response);
assertEquals("Should return a NOT_FOUND http code", URIResolver.NOT_FOUND, code);
}
/**
* Test that looking up a known URI gives a positive response code.
*
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testLookupExistingObject() throws IOException, InterruptedException {
TestResponse response = new TestResponse();
int code = aaa.lookup(new TestRequest(GIF_URL), response);
assertEquals("Should return a FOUND http code", 200, code);
assertEquals("Should have a location header", 1, response.names.size());
assertEquals("Should have a location header", 1, response.values.size());
assertEquals("Should have a location header", "Location", response.names.get(0));
assertEquals("Should have a location header", GIF_URL_KEY.getFile().toString(), response.values.get(0));
}
/**
* Test that AAA checks for null returns from the controller. This can happen if a CDX file is not backed by an arc
* file, or arc repos times out.
*
* @throws Exception
*/
@Test
public void testNullControllerReturn() throws Exception {
LogbackRecorder lr = LogbackRecorder.startRecorder();
Response response = new TestResponse();
ARCArchiveAccess nullaaa = new ARCArchiveAccess(nullArcRepos);
nullaaa.setIndex(TestInfo.ZIPPED_INDEX_DIR);
try {
nullaaa.lookup(new TestRequest(GIF_URL), response);
fail("Should throw exception if file is in index but gets null");
} catch (IOFailure e) {
// expected
}
lr.assertLogContains("Bitarchive problem must be reported in the log", "ARC file '" + GIF_URL_KEY.getFile()
+ "' mentioned in index file was not found");
lr.stopRecorder();
}
/**
* Fake arc repository client which on get returns a fake record which is ok.
*/
private class TestArcRepositoryClient extends dk.netarkivet.common.arcrepository.TestArcRepositoryClient {
public TestArcRepositoryClient() {
super(TestInfo.WORKING_DIR);
}
/**
* Returns an OK BitarchiveRecord. Content is simply arcfile name and index encoded in a stream.
*/
public BitarchiveRecord get(String arcFile, long index) {
final Map<String, Object> metadata = new HashMap<String, Object>();
for (String header_field : (List<String>) ARCConstants.REQUIRED_VERSION_1_HEADER_FIELDS) {
metadata.put(header_field, "");
}
byte[] data = ("HTTP/1.1 200 OK\nLocation: " + arcFile + "\n\n" + arcFile + " " + index).getBytes();
// TODO remove or replace this by something else ? (ARCConstants.LENGTH_HEADER_FIELD_KEY)
// does not exist in Heritrix 1.10+
// metadata.put(ARCConstants.LENGTH_HEADER_FIELD_KEY,
// Integer.toString(data.length));
// Note: ARCRecordMetadata.getLength() now reads the contents of the LENGTH_FIELD_KEY
// instead of LENGTH_HEADER_FIELD_KEY
metadata.put(ARCConstants.LENGTH_FIELD_KEY, Integer.toString(data.length));
Long dummyOffset = Long.valueOf(0L); // Note: offset is not stored as a String, but as a Long
metadata.put(ARCConstants.ABSOLUTE_OFFSET_KEY, dummyOffset);
try {
ARCRecordMetaData meta = new ARCRecordMetaData(arcFile, metadata);
return new BitarchiveRecord(new ARCRecord(new ByteArrayInputStream(data), meta), arcFile);
} catch (IOException e) {
fail("Cant't create metadata record");
return null;
}
}
}
/**
* Fake arc repository client which on get returns a fake record which is null.
*/
private class NullArcRepositoryClient extends dk.netarkivet.common.arcrepository.TestArcRepositoryClient {
public NullArcRepositoryClient() {
super(TestInfo.WORKING_DIR);
}
public BitarchiveRecord get(String arcFile, long index) {
return null;
}
}
private static class TestResponse implements Response {
ByteArrayOutputStream bas = new java.io.ByteArrayOutputStream();
int statusInt = 0;
String reasonStr = "";
List<String> names = new ArrayList<String>();
List<String> values = new ArrayList<String>();
public TestResponse() {
}
public OutputStream getOutputStream() {
return bas;
}
public void setStatus(int statusCode) {
statusInt = statusCode;
}
public void setStatus(int statusCode, String reason) {
statusInt = statusCode;
reasonStr = reason;
}
public void addHeaderField(String name, String value) {
names.add(name);
values.add(value);
}
public int getStatus() {
return statusInt;
}
}
public class TestRequest implements Request {
private URI uri;
public TestRequest(URI uri) {
this.uri = uri;
}
public URI getURI() {
return uri;
}
public Map<String, String[]> getParameterMap() {
return Collections.emptyMap();
}
}
}