Skip to content

Commit

Permalink
Update more integration tests.
Browse files Browse the repository at this point in the history
  • Loading branch information
@calvinjia
calvinjia committed Jan 7, 2016
1 parent de7cc40 commit 863d7d0
Show file tree
Hide file tree
Showing 5 changed files with 184 additions and 210 deletions.
67 changes: 30 additions & 37 deletions integration-tests/src/test/java/tachyon/client/FileOutStreamIntegrationTest.java
View file
Expand Up @@ -31,13 +31,11 @@
import tachyon.TachyonURI; import tachyon.TachyonURI;
import tachyon.client.file.FileInStream; import tachyon.client.file.FileInStream;
import tachyon.client.file.FileOutStream; import tachyon.client.file.FileOutStream;
import tachyon.client.file.TachyonFile;
import tachyon.client.file.FileSystem; import tachyon.client.file.FileSystem;
import tachyon.client.file.options.InStreamOptions; import tachyon.client.file.URIStatus;
import tachyon.client.file.options.OutStreamOptions; import tachyon.client.file.options.CreateFileOptions;
import tachyon.conf.TachyonConf; import tachyon.conf.TachyonConf;
import tachyon.exception.TachyonException; import tachyon.exception.TachyonException;
import tachyon.thrift.FileInfo;
import tachyon.underfs.UnderFileSystem; import tachyon.underfs.UnderFileSystem;
import tachyon.underfs.UnderFileSystemCluster; import tachyon.underfs.UnderFileSystemCluster;
import tachyon.util.io.BufferUtils; import tachyon.util.io.BufferUtils;
Expand All @@ -62,10 +60,10 @@ public final class FileOutStreamIntegrationTest {
Constants.USER_FILE_BUFFER_BYTES, String.valueOf(BUFFER_BYTES), Constants.USER_FILE_BUFFER_BYTES, String.valueOf(BUFFER_BYTES),
Constants.WORKER_DATA_SERVER, IntegrationTestConstants.NETTY_DATA_SERVER); Constants.WORKER_DATA_SERVER, IntegrationTestConstants.NETTY_DATA_SERVER);


private OutStreamOptions mWriteBoth; private CreateFileOptions mWriteBoth;
private OutStreamOptions mWriteTachyon; private CreateFileOptions mWriteTachyon;
private OutStreamOptions mWriteUnderStore; private CreateFileOptions mWriteUnderStore;
private OutStreamOptions mWriteLocal; private CreateFileOptions mWriteLocal;
private TachyonConf mTestConf; private TachyonConf mTestConf;
private FileSystem mTfs = null; private FileSystem mTfs = null;


Expand All @@ -90,33 +88,28 @@ public final void before() throws Exception {
*/ */
private void checkWrite(TachyonURI filePath, UnderStorageType underStorageType, int fileLen, private void checkWrite(TachyonURI filePath, UnderStorageType underStorageType, int fileLen,
int increasingByteArrayLen) throws IOException, TachyonException { int increasingByteArrayLen) throws IOException, TachyonException {
for (OutStreamOptions op : getOptionSet()) { for (CreateFileOptions op : getOptionSet()) {
TachyonFile file = mTfs.open(filePath); URIStatus status = mTfs.getStatus(filePath);
FileInfo info = mTfs.getInfo(file); Assert.assertEquals(fileLen, status.getLength());
Assert.assertEquals(fileLen, info.getLength()); FileInStream is = mTfs.openFile(filePath, TachyonFSTestUtils.toOpenFileOptions(op));
InStreamOptions op2 = byte[] res = new byte[(int) status.getLength()];
new InStreamOptions.Builder(mTestConf).setTachyonStorageType( Assert.assertEquals((int) status.getLength(), is.read(res));
op.getTachyonStorageType()).build();
FileInStream is = mTfs.getInStream(file, op2);
byte[] res = new byte[(int) info.getLength()];
Assert.assertEquals((int) info.getLength(), is.read(res));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(increasingByteArrayLen, res)); Assert.assertTrue(BufferUtils.equalIncreasingByteArray(increasingByteArrayLen, res));
is.close(); is.close();
} }


if (underStorageType.isSyncPersist()) { if (underStorageType.isSyncPersist()) {
TachyonFile file = mTfs.open(filePath); URIStatus status = mTfs.getStatus(filePath);
FileInfo info = mTfs.getInfo(file); String checkpointPath = status.getUfsPath();
String checkpointPath = info.getUfsPath();
UnderFileSystem ufs = UnderFileSystem.get(checkpointPath, mTestConf); UnderFileSystem ufs = UnderFileSystem.get(checkpointPath, mTestConf);


InputStream is = ufs.open(checkpointPath); InputStream is = ufs.open(checkpointPath);
byte[] res = new byte[(int) info.getLength()]; byte[] res = new byte[(int) status.getLength()];
if (UnderFileSystemCluster.readEOFReturnsNegative() && 0 == res.length) { if (UnderFileSystemCluster.readEOFReturnsNegative() && 0 == res.length) {
// Returns -1 for zero-sized byte array to indicate no more bytes available here. // Returns -1 for zero-sized byte array to indicate no more bytes available here.
Assert.assertEquals(-1, is.read(res)); Assert.assertEquals(-1, is.read(res));
} else { } else {
Assert.assertEquals((int) info.getLength(), is.read(res)); Assert.assertEquals((int) status.getLength(), is.read(res));
} }
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(increasingByteArrayLen, res)); Assert.assertTrue(BufferUtils.equalIncreasingByteArray(increasingByteArrayLen, res));
is.close(); is.close();
Expand All @@ -130,15 +123,15 @@ private void checkWrite(TachyonURI filePath, UnderStorageType underStorageType,
public void writeTest1() throws IOException, TachyonException { public void writeTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath(); String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) { for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (OutStreamOptions op : getOptionSet()) { for (CreateFileOptions op : getOptionSet()) {
writeTest1Util(new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode()), k, op); writeTest1Util(new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode()), k, op);
} }
} }
} }


private void writeTest1Util(TachyonURI filePath, int len, OutStreamOptions op) private void writeTest1Util(TachyonURI filePath, int len, CreateFileOptions op)
throws IOException, TachyonException { throws IOException, TachyonException {
FileOutStream os = mTfs.getOutStream(filePath, op); FileOutStream os = mTfs.createFile(filePath, op);
for (int k = 0; k < len; k ++) { for (int k = 0; k < len; k ++) {
os.write((byte) k); os.write((byte) k);
} }
Expand All @@ -153,15 +146,15 @@ private void writeTest1Util(TachyonURI filePath, int len, OutStreamOptions op)
public void writeTest2() throws IOException, TachyonException { public void writeTest2() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath(); String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) { for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (OutStreamOptions op : getOptionSet()) { for (CreateFileOptions op : getOptionSet()) {
writeTest2Util(new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode()), k, op); writeTest2Util(new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode()), k, op);
} }
} }
} }


private void writeTest2Util(TachyonURI filePath, int len, OutStreamOptions op) private void writeTest2Util(TachyonURI filePath, int len, CreateFileOptions op)
throws IOException, TachyonException { throws IOException, TachyonException {
FileOutStream os = mTfs.getOutStream(filePath, op); FileOutStream os = mTfs.createFile(filePath, op);
os.write(BufferUtils.getIncreasingByteArray(len)); os.write(BufferUtils.getIncreasingByteArray(len));
os.close(); os.close();
checkWrite(filePath, op.getUnderStorageType(), len, len); checkWrite(filePath, op.getUnderStorageType(), len, len);
Expand All @@ -174,15 +167,15 @@ private void writeTest2Util(TachyonURI filePath, int len, OutStreamOptions op)
public void writeTest3() throws IOException, TachyonException { public void writeTest3() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath(); String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) { for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (OutStreamOptions op : getOptionSet()) { for (CreateFileOptions op : getOptionSet()) {
writeTest3Util(new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode()), k, op); writeTest3Util(new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode()), k, op);
} }
} }
} }


private void writeTest3Util(TachyonURI filePath, int len, OutStreamOptions op) private void writeTest3Util(TachyonURI filePath, int len, CreateFileOptions op)
throws IOException, TachyonException { throws IOException, TachyonException {
FileOutStream os = mTfs.getOutStream(filePath, op); FileOutStream os = mTfs.createFile(filePath, op);
os.write(BufferUtils.getIncreasingByteArray(0, len / 2), 0, len / 2); os.write(BufferUtils.getIncreasingByteArray(0, len / 2), 0, len / 2);
os.write(BufferUtils.getIncreasingByteArray(len / 2, len / 2), 0, len / 2); os.write(BufferUtils.getIncreasingByteArray(len / 2, len / 2), 0, len / 2);
os.close(); os.close();
Expand All @@ -198,7 +191,7 @@ private void writeTest3Util(TachyonURI filePath, int len, OutStreamOptions op)
public void writeSpecifyLocalTest() throws IOException, TachyonException { public void writeSpecifyLocalTest() throws IOException, TachyonException {
TachyonURI filePath = new TachyonURI(PathUtils.uniqPath()); TachyonURI filePath = new TachyonURI(PathUtils.uniqPath());
final int length = 2; final int length = 2;
FileOutStream os = mTfs.getOutStream(filePath, mWriteLocal); FileOutStream os = mTfs.createFile(filePath, mWriteLocal);
os.write((byte) 0); os.write((byte) 0);
os.write((byte) 1); os.write((byte) 1);
os.close(); os.close();
Expand All @@ -217,7 +210,7 @@ public void longWriteTest() throws IOException, InterruptedException,
TachyonException { TachyonException {
TachyonURI filePath = new TachyonURI(PathUtils.uniqPath()); TachyonURI filePath = new TachyonURI(PathUtils.uniqPath());
final int length = 2; final int length = 2;
FileOutStream os = mTfs.getOutStream(filePath, mWriteUnderStore); FileOutStream os = mTfs.createFile(filePath, mWriteUnderStore);
os.write((byte) 0); os.write((byte) 0);
Thread.sleep(mTestConf.getInt(Constants.USER_HEARTBEAT_INTERVAL_MS) * 2); Thread.sleep(mTestConf.getInt(Constants.USER_HEARTBEAT_INTERVAL_MS) * 2);
os.write((byte) 1); os.write((byte) 1);
Expand All @@ -235,7 +228,7 @@ public void longWriteTest() throws IOException, InterruptedException,
@Test @Test
public void outOfOrderWriteTest() throws IOException, TachyonException { public void outOfOrderWriteTest() throws IOException, TachyonException {
TachyonURI filePath = new TachyonURI(PathUtils.uniqPath()); TachyonURI filePath = new TachyonURI(PathUtils.uniqPath());
FileOutStream os = mTfs.getOutStream(filePath, mWriteTachyon); FileOutStream os = mTfs.createFile(filePath, mWriteTachyon);


// Write something small, so it is written into the buffer, and not directly to the file. // Write something small, so it is written into the buffer, and not directly to the file.
os.write((byte) 0); os.write((byte) 0);
Expand All @@ -250,8 +243,8 @@ public void outOfOrderWriteTest() throws IOException, TachyonException {
checkWrite(filePath, mWriteTachyon.getUnderStorageType(), length + 1, length + 1); checkWrite(filePath, mWriteTachyon.getUnderStorageType(), length + 1, length + 1);
} }


private List<OutStreamOptions> getOptionSet() { private List<CreateFileOptions> getOptionSet() {
List<OutStreamOptions> ret = new ArrayList<OutStreamOptions>(3); List<CreateFileOptions> ret = new ArrayList<CreateFileOptions>(3);
ret.add(mWriteBoth); ret.add(mWriteBoth);
ret.add(mWriteTachyon); ret.add(mWriteTachyon);
ret.add(mWriteUnderStore); ret.add(mWriteUnderStore);
Expand Down

0 comments on commit 863d7d0

Please sign in to comment.