Permalink
Browse files

Allow PackFile to lazily load its PackIndex on first demand

Repositories which contain several pack files may have a very large
historical pack holding the bulk of the project data, and a much
smaller more recent pack holding current data.  In such cases we
may be able to answer queries entirely by reading the recent pack
index, and never reading the larger historical index.  Since JGit
reads the entire index into memory, avoiding reading the larger
index can save some time on short uses of a Repository.

This change only sets up the lazy loading of the index.  Because
the packs aren't sorted by newest-to-oldest in Repository we are
still very likely to demand load packs we didn't need to examine.

This change also opens the door for potentially releasing one or
more PackIndex objects when memory gets low, but doesn't actually
implement any sort of soft reference semantics.

Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Robin Rosenberg <robin.rosenberg@dewire.com>
  • Loading branch information...
spearce authored and robinrosenberg committed Feb 12, 2009
1 parent c438ba9 commit 7545f933f9da5fee7939600022c9271dfb705c26
@@ -42,6 +42,7 @@
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.Collections;
import java.util.Iterator;
import java.util.zip.CRC32;
import java.util.zip.CheckedOutputStream;
@@ -56,9 +57,11 @@
* objects are similar.
*/
public class PackFile implements Iterable<PackIndex.MutableEntry> {
+ private final File idxFile;
+
private final WindowedFile pack;
- private final PackIndex idx;
+ private PackIndex loadedIdx;
private PackReverseIndex reverseIdx;
@@ -69,21 +72,22 @@
* path of the <code>.idx</code> file listing the contents.
* @param packFile
* path of the <code>.pack</code> file holding the data.
- * @throws IOException
- * the index file cannot be accessed at this time.
*/
- public PackFile(final File idxFile, final File packFile) throws IOException {
+ public PackFile(final File idxFile, final File packFile) {
+ this.idxFile = idxFile;
pack = new WindowedFile(packFile) {
@Override
protected void onOpen() throws IOException {
readPackHeader();
}
};
- try {
- idx = PackIndex.open(idxFile);
- } catch (IOException ioe) {
- throw ioe;
+ }
+
+ private synchronized PackIndex idx() throws IOException {
+ if (loadedIdx == null) {
+ loadedIdx = PackIndex.open(idxFile);
}
+ return loadedIdx;
}
final PackedObjectLoader resolveBase(final WindowCursor curs, final long ofs)
@@ -106,9 +110,11 @@ public File getPackFile() {
* @param id
* the object to look for. Must not be null.
* @return true if the object is in this pack; false otherwise.
+ * @throws IOException
+ * the index file cannot be loaded into memory.
*/
- public boolean hasObject(final AnyObjectId id) {
- return idx.hasObject(id);
+ public boolean hasObject(final AnyObjectId id) throws IOException {
+ return idx().hasObject(id);
}
/**
@@ -125,7 +131,7 @@ public boolean hasObject(final AnyObjectId id) {
*/
public PackedObjectLoader get(final WindowCursor curs, final AnyObjectId id)
throws IOException {
- final long offset = idx.findOffset(id);
+ final long offset = idx().findOffset(id);
return 0 < offset ? reader(curs, offset) : null;
}
@@ -135,6 +141,9 @@ public PackedObjectLoader get(final WindowCursor curs, final AnyObjectId id)
public void close() {
UnpackedObjectCache.purge(pack);
pack.close();
+ synchronized (this) {
+ loadedIdx = null;
+ }
}
/**
@@ -150,17 +159,23 @@ public void close() {
* @see PackIndex#iterator()
*/
public Iterator<PackIndex.MutableEntry> iterator() {
- return idx.iterator();
+ try {
+ return idx().iterator();
+ } catch (IOException e) {
+ return Collections.<PackIndex.MutableEntry> emptyList().iterator();
+ }
}
/**
* Obtain the total number of objects available in this pack. This method
* relies on pack index, giving number of effectively available objects.
*
* @return number of objects in index of this pack, likewise in this pack
+ * @throws IOException
+ * the index file cannot be loaded into memory.
*/
- long getObjectCount() {
- return idx.getObjectCount();
+ long getObjectCount() throws IOException {
+ return idx().getObjectCount();
}
/**
@@ -170,8 +185,10 @@ long getObjectCount() {
* @param offset
* start offset of object to find
* @return object id for this offset, or null if no object was found
+ * @throws IOException
+ * the index file cannot be loaded into memory.
*/
- ObjectId findObjectForOffset(final long offset) {
+ ObjectId findObjectForOffset(final long offset) throws IOException {
return getReverseIdx().findObject(offset);
}
@@ -196,6 +213,7 @@ final void copyRawData(final PackedObjectLoader loader,
final long dataOffset = loader.dataOffset;
final int cnt = (int) (findEndOffset(objectOffset) - dataOffset);
final WindowCursor curs = loader.curs;
+ final PackIndex idx = idx();
if (idx.hasCRC32Support()) {
final CRC32 crc = new CRC32();
@@ -231,8 +249,8 @@ final void copyRawData(final PackedObjectLoader loader,
}
}
- boolean supportsFastCopyRawData() {
- return idx.hasCRC32Support();
+ boolean supportsFastCopyRawData() throws IOException {
+ return idx().hasCRC32Support();
}
@@ -258,11 +276,12 @@ private void readPackHeader() throws IOException {
position += 4;
pack.readFully(position, intbuf, curs);
- final long objectCnt = NB.decodeUInt32(intbuf, 0);
- if (idx.getObjectCount() != objectCnt)
+ final long packCnt = NB.decodeUInt32(intbuf, 0);
+ final long idxCnt = idx().getObjectCount();
+ if (idxCnt != packCnt)
throw new IOException("Pack index"
- + " object count mismatch; expected " + objectCnt
- + " found " + idx.getObjectCount() + ": " + pack.getName());
+ + " object count mismatch; expected " + packCnt
+ + " found " + idxCnt + ": " + pack.getName());
}
private PackedObjectLoader reader(final WindowCursor curs,
@@ -315,14 +334,14 @@ private PackedObjectLoader reader(final WindowCursor curs,
}
private long findEndOffset(final long startOffset)
- throws CorruptObjectException {
+ throws IOException, CorruptObjectException {
final long maxOffset = pack.length() - Constants.OBJECT_ID_LENGTH;
return getReverseIdx().findNextOffset(startOffset, maxOffset);
}
- private synchronized PackReverseIndex getReverseIdx() {
+ private synchronized PackReverseIndex getReverseIdx() throws IOException {
if (reverseIdx == null)
- reverseIdx = new PackReverseIndex(idx);
+ reverseIdx = new PackReverseIndex(idx());
return reverseIdx;
}
}
@@ -103,8 +103,10 @@ public void copyRawData(OutputStream out, byte buf[]) throws IOException {
* @return true if this loader is capable of fast raw-data copying basing on
* compressed data checksum; false if raw-data copying needs
* uncompressing and compressing data
+ * @throws IOException
+ * the index file format cannot be determined.
*/
- public boolean supportsFastCopyRawData() {
+ public boolean supportsFastCopyRawData() throws IOException {
return pack.supportsFastCopyRawData();
}
@@ -248,8 +248,15 @@ public boolean hasObject(final AnyObjectId objectId) {
final PackFile[] packs = packs();
int k = packs.length;
while (k > 0) {
- if (packs[--k].hasObject(objectId))
- return true;
+ try {
+ if (packs[--k].hasObject(objectId))
+ return true;
+ } catch (IOException e) {
+ // Assume that means the pack is invalid, and such
+ // packs are treated as though they are empty.
+ //
+ continue;
+ }
}
return toFile(objectId).isFile();
}
@@ -848,13 +855,7 @@ public boolean accept(final File baseDir, final String n) {
continue SCAN;
}
- try {
- packList.add(new PackFile(idxFile, packFile));
- } catch (IOException ioe) {
- // Whoops. That's not a pack!
- //
- ioe.printStackTrace();
- }
+ packList.add(new PackFile(idxFile, packFile));
}
}
}

0 comments on commit 7545f93

Please sign in to comment.