Skip to content

Commit

Permalink
Cache support unbounded requests.
Browse files Browse the repository at this point in the history
-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=131696858
  • Loading branch information
erdemguven authored and ojw28 committed Aug 31, 2016
1 parent dfad745 commit bd7be1b
Show file tree
Hide file tree
Showing 11 changed files with 579 additions and 85 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.upstream.cache;

import android.net.Uri;
import android.test.InstrumentationTestCase;
import android.test.MoreAsserts;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.testutil.FakeDataSource;
import com.google.android.exoplayer2.testutil.FakeDataSource.Builder;
import com.google.android.exoplayer2.testutil.TestUtil;
import com.google.android.exoplayer2.upstream.DataSpec;
import com.google.android.exoplayer2.upstream.HttpDataSource.InvalidResponseCodeException;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;

/** Unit tests for {@link CacheDataSource}. */
public class CacheDataSourceTest extends InstrumentationTestCase {

private static final byte[] TEST_DATA = new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
private static final int MAX_CACHE_FILE_SIZE = 3;
private static final String KEY_1 = "key 1";
private static final String KEY_2 = "key 2";

private File cacheDir;
private SimpleCache simpleCache;

@Override
protected void setUp() throws Exception {
// Create a temporary folder
cacheDir = File.createTempFile("CacheDataSourceTest", null);
assertTrue(cacheDir.delete());
assertTrue(cacheDir.mkdir());

simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
}

@Override
protected void tearDown() throws Exception {
TestUtil.recursiveDelete(cacheDir);
}

public void testMaxCacheFileSize() throws Exception {
CacheDataSource cacheDataSource = createCacheDataSource(false, false, false);
assertReadDataContentLength(cacheDataSource, false, false);
assertEquals((int) Math.ceil((double) TEST_DATA.length / MAX_CACHE_FILE_SIZE),
cacheDir.listFiles().length);
}

public void testCacheAndRead() throws Exception {
assertCacheAndRead(false, false);
}

public void testCacheAndReadUnboundedRequest() throws Exception {
assertCacheAndRead(true, false);
}

public void testCacheAndReadUnknownLength() throws Exception {
assertCacheAndRead(false, true);
}

// Disabled test as we don't support caching of definitely unknown length content
public void disabledTestCacheAndReadUnboundedRequestUnknownLength() throws Exception {
assertCacheAndRead(true, true);
}

public void testUnsatisfiableRange() throws Exception {
// Bounded request but the content length is unknown. This forces all data to be cached but not
// the length
assertCacheAndRead(false, true);

// Now do an unbounded request. This will read all of the data from cache and then try to read
// more from upstream which will cause to a 416 so CDS will store the length.
CacheDataSource cacheDataSource = createCacheDataSource(true, true, true);
assertReadDataContentLength(cacheDataSource, true, true);

// If the user try to access off range then it should throw an IOException
try {
cacheDataSource = createCacheDataSource(false, false, false);
cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length, 5, KEY_1));
fail();
} catch (TestIOException e) {
// success
}
}

public void testContentLengthEdgeCases() throws Exception {
// Read partial at EOS but don't cross it so length is unknown
CacheDataSource cacheDataSource = createCacheDataSource(false, false, true);
assertReadData(cacheDataSource, true, TEST_DATA.length - 2, 2);
assertEquals(C.LENGTH_UNSET, simpleCache.getContentLength(KEY_1));

// Now do an unbounded request for whole data. This will cause a bounded request from upstream.
// End of data from upstream shouldn't be mixed up with EOS and cause length set wrong.
cacheDataSource = createCacheDataSource(true, false, true);
assertReadDataContentLength(cacheDataSource, true, true);

// Now the length set correctly do an unbounded request with offset
assertEquals(2, cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length - 2,
C.LENGTH_UNSET, KEY_1)));

// An unbounded request with offset for not cached content
assertEquals(C.LENGTH_UNSET, cacheDataSource.open(new DataSpec(Uri.EMPTY, TEST_DATA.length - 2,
C.LENGTH_UNSET, KEY_2)));
}

private void assertCacheAndRead(boolean unboundedRequest, boolean simulateUnknownLength)
throws IOException {
// Read all data from upstream and cache
CacheDataSource cacheDataSource = createCacheDataSource(false, false, simulateUnknownLength);
assertReadDataContentLength(cacheDataSource, unboundedRequest, simulateUnknownLength);

// Just read from cache
cacheDataSource = createCacheDataSource(false, true, simulateUnknownLength);
assertReadDataContentLength(cacheDataSource, unboundedRequest,
false /*length is already cached*/);
}

/**
* Reads data until EOI and compares it to {@link #TEST_DATA}. Also checks content length returned
* from open() call and the cached content length.
*/
private void assertReadDataContentLength(CacheDataSource cacheDataSource,
boolean unboundedRequest, boolean unknownLength) throws IOException {
int length = unboundedRequest ? C.LENGTH_UNSET : TEST_DATA.length;
assertReadData(cacheDataSource, unknownLength, 0, length);
assertEquals("When the range specified, CacheDataSource doesn't reach EOS so shouldn't cache "
+ "content length", !unboundedRequest ? C.LENGTH_UNSET : TEST_DATA.length,
simpleCache.getContentLength(KEY_1));
}

private void assertReadData(CacheDataSource cacheDataSource, boolean unknownLength, int position,
int length) throws IOException {
int actualLength = TEST_DATA.length - position;
if (length != C.LENGTH_UNSET) {
actualLength = Math.min(actualLength, length);
}
assertEquals(unknownLength ? length : actualLength,
cacheDataSource.open(new DataSpec(Uri.EMPTY, position, length, KEY_1)));

byte[] buffer = new byte[100];
int index = 0;
while (true) {
int read = cacheDataSource.read(buffer, index, buffer.length - index);
if (read == C.RESULT_END_OF_INPUT) {
break;
}
index += read;
}
assertEquals(actualLength, index);
MoreAsserts.assertEquals(Arrays.copyOfRange(TEST_DATA, position, position + actualLength),
Arrays.copyOf(buffer, index));

cacheDataSource.close();
}

private CacheDataSource createCacheDataSource(boolean set416exception, boolean setReadException,
boolean simulateUnknownLength) {
Builder builder = new Builder();
if (setReadException) {
builder.appendReadError(new IOException("Shouldn't read from upstream"));
}
builder.setSimulateUnknownLength(simulateUnknownLength);
builder.appendReadData(TEST_DATA);
FakeDataSource upstream = builder.build();
upstream.setUnsatisfiableRangeException(set416exception
? new InvalidResponseCodeException(416, null, null)
: new TestIOException());
return new CacheDataSource(simpleCache, upstream,
CacheDataSource.FLAG_BLOCK_ON_CACHE | CacheDataSource.FLAG_CACHE_UNBOUNDED_REQUESTS,
MAX_CACHE_FILE_SIZE);
}

private static class TestIOException extends IOException {}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.upstream.cache;

import android.test.InstrumentationTestCase;

import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.testutil.TestUtil;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.NavigableSet;
import java.util.Set;

/**
* Unit tests for {@link SimpleCache}.
*/
public class SimpleCacheTest extends InstrumentationTestCase {

private static final String KEY_1 = "key1";

private File cacheDir;

@Override
protected void setUp() throws Exception {
// Create a temporary folder
cacheDir = File.createTempFile("SimpleCacheTest", null);
assertTrue(cacheDir.delete());
assertTrue(cacheDir.mkdir());
}

@Override
protected void tearDown() throws Exception {
TestUtil.recursiveDelete(cacheDir);
}

public void testCommittingOneFile() throws Exception {
SimpleCache simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());

CacheSpan cacheSpan = simpleCache.startReadWrite(KEY_1, 0);
assertFalse(cacheSpan.isCached);
assertTrue(cacheSpan.isOpenEnded());

assertNull(simpleCache.startReadWriteNonBlocking(KEY_1, 0));

assertEquals(0, simpleCache.getKeys().size());
NavigableSet<CacheSpan> cachedSpans = simpleCache.getCachedSpans(KEY_1);
assertTrue(cachedSpans == null || cachedSpans.size() == 0);
assertEquals(0, simpleCache.getCacheSpace());
assertEquals(0, cacheDir.listFiles().length);

addCache(simpleCache, 0, 15);

Set<String> cachedKeys = simpleCache.getKeys();
assertEquals(1, cachedKeys.size());
assertTrue(cachedKeys.contains(KEY_1));
cachedSpans = simpleCache.getCachedSpans(KEY_1);
assertEquals(1, cachedSpans.size());
assertTrue(cachedSpans.contains(cacheSpan));
assertEquals(15, simpleCache.getCacheSpace());

cacheSpan = simpleCache.startReadWrite(KEY_1, 0);
assertTrue(cacheSpan.isCached);
assertFalse(cacheSpan.isOpenEnded());
assertEquals(15, cacheSpan.length);
}

public void testSetGetLength() throws Exception {
SimpleCache simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());

assertEquals(C.LENGTH_UNSET, simpleCache.getContentLength(KEY_1));
assertTrue(simpleCache.setContentLength(KEY_1, 15));
assertEquals(15, simpleCache.getContentLength(KEY_1));

simpleCache.startReadWrite(KEY_1, 0);

addCache(simpleCache, 0, 15);

assertTrue(simpleCache.setContentLength(KEY_1, 150));
assertEquals(150, simpleCache.getContentLength(KEY_1));

addCache(simpleCache, 140, 10);

// Try to set length shorter then the content
assertFalse(simpleCache.setContentLength(KEY_1, 15));
assertEquals("Content length should be unchanged.",
150, simpleCache.getContentLength(KEY_1));

/* TODO Enable when the length persistance is fixed
// Check if values are kept after cache is reloaded.
simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
assertEquals(150, simpleCache.getContentLength(KEY_1));
CacheSpan lastSpan = simpleCache.startReadWrite(KEY_1, 145);
// Removing the last span shouldn't cause the length be change next time cache loaded
simpleCache.removeSpan(lastSpan);
simpleCache = new SimpleCache(cacheDir, new NoOpCacheEvictor());
assertEquals(150, simpleCache.getContentLength(KEY_1));
*/
}

private void addCache(SimpleCache simpleCache, int position, int length) throws IOException {
File file = simpleCache.startFile(KEY_1, position, length);
FileOutputStream fos = new FileOutputStream(file);
fos.write(new byte[length]);
fos.close();
simpleCache.commitFile(file);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -143,11 +143,11 @@ interface Listener {
*
* @param key The cache key for the data.
* @param position The starting position of the data.
* @param length The length of the data to be written. Used only to ensure that there is enough
* space in the cache.
* @param maxLength The maximum length of the data to be written. Used only to ensure that there
* is enough space in the cache.
* @return The file into which data should be written.
*/
File startFile(String key, long position, long length);
File startFile(String key, long position, long maxLength);

/**
* Commits a file into the cache. Must only be called when holding a corresponding hole
Expand Down Expand Up @@ -182,4 +182,22 @@ interface Listener {
*/
boolean isCached(String key, long position, long length);

/**
* Sets the content length for the given key.
*
* @param key The cache key for the data.
* @param length The length of the data.
* @return Whether the length was set successfully. Returns false if the length conflicts with the
* existing contents of the cache.
*/
boolean setContentLength(String key, long length);

/**
* Returns the content length for the given key if one set, or {@link
* com.google.android.exoplayer2.C#LENGTH_UNSET} otherwise.
*
* @param key The cache key for the data.
*/
long getContentLength(String key);

}
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,12 @@ public CacheDataSink(Cache cache, long maxCacheFileSize) {

@Override
public void open(DataSpec dataSpec) throws CacheDataSinkException {
// TODO: Support caching for unbounded requests. See TODO in {@link CacheDataSource} for
// more details.
Assertions.checkState(dataSpec.length != C.LENGTH_UNSET);
this.dataSpec = dataSpec;
if (dataSpec.length == C.LENGTH_UNSET) {
return;
}
dataSpecBytesWritten = 0;
try {
this.dataSpec = dataSpec;
dataSpecBytesWritten = 0;
openNextOutputStream();
} catch (FileNotFoundException e) {
throw new CacheDataSinkException(e);
Expand All @@ -78,6 +78,9 @@ public void open(DataSpec dataSpec) throws CacheDataSinkException {

@Override
public void write(byte[] buffer, int offset, int length) throws CacheDataSinkException {
if (dataSpec.length == C.LENGTH_UNSET) {
return;
}
try {
int bytesWritten = 0;
while (bytesWritten < length) {
Expand All @@ -99,6 +102,9 @@ public void write(byte[] buffer, int offset, int length) throws CacheDataSinkExc

@Override
public void close() throws CacheDataSinkException {
if (dataSpec == null || dataSpec.length == C.LENGTH_UNSET) {
return;
}
try {
closeCurrentOutputStream();
} catch (IOException e) {
Expand Down
Loading

0 comments on commit bd7be1b

Please sign in to comment.