From e33f2e01973f4d82dff8fad3a1d96e9037f4bdbb Mon Sep 17 00:00:00 2001 From: eolivelli Date: Wed, 25 Nov 2015 17:07:13 +0100 Subject: [PATCH 1/6] BOOKKEEPER-879: Record ledger creation time --- .../bookkeeper/client/LedgerMetadata.java | 19 +++- .../apache/bookkeeper/proto/DataFormats.java | 86 ++++++++++++++++--- .../src/main/proto/DataFormats.proto | 2 + .../bookkeeper/client/ListLedgersTest.java | 54 ++++++++---- 4 files changed, 129 insertions(+), 32 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java index a20f34ade6a..1e153dd1519 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java @@ -66,6 +66,7 @@ public class LedgerMetadata { private int ackQuorumSize; private long length; private long lastEntryId; + private long ctime; private LedgerMetadataFormat.State state; private SortedMap> ensembles = @@ -82,6 +83,7 @@ public LedgerMetadata(int ensembleSize, int writeQuorumSize, int ackQuorumSize, this.ensembleSize = ensembleSize; this.writeQuorumSize = writeQuorumSize; this.ackQuorumSize = ackQuorumSize; + this.ctime = System.currentTimeMillis(); /* * It is set in PendingReadOp.readEntryComplete, and @@ -104,6 +106,7 @@ public LedgerMetadata(int ensembleSize, int writeQuorumSize, int ackQuorumSize, LedgerMetadata(LedgerMetadata other) { this.ensembleSize = other.ensembleSize; this.writeQuorumSize = other.writeQuorumSize; + this.ctime = other.ctime; this.ackQuorumSize = other.ackQuorumSize; this.length = other.length; this.lastEntryId = other.lastEntryId; @@ -111,7 +114,7 @@ public LedgerMetadata(int ensembleSize, int writeQuorumSize, int ackQuorumSize, this.state = other.state; this.version = other.version; this.hasPassword = other.hasPassword; - this.digestType = other.digestType; + this.digestType = other.digestType; this.password = new byte[other.password.length]; System.arraycopy(other.password, 0, this.password, 0, other.password.length); // copy the ensembles @@ -150,6 +153,14 @@ public int getWriteQuorumSize() { return writeQuorumSize; } + /** + * Get the creation timestamp of the ledger + * @return + */ + public long getCtime() { + return ctime; + } + public int getAckQuorumSize() { return ackQuorumSize; } @@ -255,7 +266,7 @@ public byte[] serialize() { LedgerMetadataFormat.Builder builder = LedgerMetadataFormat.newBuilder(); builder.setQuorumSize(writeQuorumSize).setAckQuorumSize(ackQuorumSize) .setEnsembleSize(ensembleSize).setLength(length) - .setState(state).setLastEntryId(lastEntryId); + .setState(state).setLastEntryId(lastEntryId).setCtime(ctime); if (hasPassword) { builder.setDigestType(digestType).setPassword(ByteString.copyFrom(password)); @@ -351,7 +362,8 @@ public static LedgerMetadata parseConfig(byte[] bytes, Version version) throws I LedgerMetadataFormat.Builder builder = LedgerMetadataFormat.newBuilder(); TextFormat.merge(reader, builder); LedgerMetadataFormat data = builder.build(); - lc.writeQuorumSize = data.getQuorumSize(); + lc.writeQuorumSize = data.getQuorumSize(); + lc.ctime=data.getCtime(); if (data.hasAckQuorumSize()) { lc.ackQuorumSize = data.getAckQuorumSize(); } else { @@ -463,6 +475,7 @@ boolean isConflictWith(LedgerMetadata newMeta) { if (metadataFormatVersion != newMeta.metadataFormatVersion || ensembleSize != newMeta.ensembleSize || writeQuorumSize != newMeta.writeQuorumSize || + ctime != newMeta.ctime || ackQuorumSize != newMeta.ackQuorumSize || length != newMeta.length || state != newMeta.state || diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java index 65d5444d10d..4552144ab9d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java @@ -52,6 +52,10 @@ org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.SegmentOrBuilder ge // optional int32 ackQuorumSize = 9; boolean hasAckQuorumSize(); int getAckQuorumSize(); + + // optional int64 ctime = 10; + boolean hasCtime(); + long getCtime(); } public static final class LedgerMetadataFormat extends com.google.protobuf.GeneratedMessage @@ -781,6 +785,16 @@ public int getAckQuorumSize() { return ackQuorumSize_; } + // optional int64 ctime = 10; + public static final int CTIME_FIELD_NUMBER = 10; + private long ctime_; + public boolean hasCtime() { + return ((bitField0_ & 0x00000100) == 0x00000100); + } + public long getCtime() { + return ctime_; + } + private void initFields() { quorumSize_ = 0; ensembleSize_ = 0; @@ -791,6 +805,7 @@ private void initFields() { digestType_ = org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.DigestType.CRC32; password_ = com.google.protobuf.ByteString.EMPTY; ackQuorumSize_ = 0; + ctime_ = 0L; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -853,6 +868,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) if (((bitField0_ & 0x00000080) == 0x00000080)) { output.writeInt32(9, ackQuorumSize_); } + if (((bitField0_ & 0x00000100) == 0x00000100)) { + output.writeInt64(10, ctime_); + } getUnknownFields().writeTo(output); } @@ -898,6 +916,10 @@ public int getSerializedSize() { size += com.google.protobuf.CodedOutputStream .computeInt32Size(9, ackQuorumSize_); } + if (((bitField0_ & 0x00000100) == 0x00000100)) { + size += com.google.protobuf.CodedOutputStream + .computeInt64Size(10, ctime_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -1045,6 +1067,8 @@ public Builder clear() { bitField0_ = (bitField0_ & ~0x00000080); ackQuorumSize_ = 0; bitField0_ = (bitField0_ & ~0x00000100); + ctime_ = 0L; + bitField0_ = (bitField0_ & ~0x00000200); return this; } @@ -1124,6 +1148,10 @@ public org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat buildPartial to_bitField0_ |= 0x00000080; } result.ackQuorumSize_ = ackQuorumSize_; + if (((from_bitField0_ & 0x00000200) == 0x00000200)) { + to_bitField0_ |= 0x00000100; + } + result.ctime_ = ctime_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -1190,6 +1218,9 @@ public Builder mergeFrom(org.apache.bookkeeper.proto.DataFormats.LedgerMetadataF if (other.hasAckQuorumSize()) { setAckQuorumSize(other.getAckQuorumSize()); } + if (other.hasCtime()) { + setCtime(other.getCtime()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -1301,6 +1332,11 @@ public Builder mergeFrom( ackQuorumSize_ = input.readInt32(); break; } + case 80: { + bitField0_ |= 0x00000200; + ctime_ = input.readInt64(); + break; + } } } } @@ -1670,6 +1706,27 @@ public Builder clearAckQuorumSize() { return this; } + // optional int64 ctime = 10; + private long ctime_ ; + public boolean hasCtime() { + return ((bitField0_ & 0x00000200) == 0x00000200); + } + public long getCtime() { + return ctime_; + } + public Builder setCtime(long value) { + bitField0_ |= 0x00000200; + ctime_ = value; + onChanged(); + return this; + } + public Builder clearCtime() { + bitField0_ = (bitField0_ & ~0x00000200); + ctime_ = 0L; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:LedgerMetadataFormat) } @@ -3993,7 +4050,7 @@ void setBookieId(com.google.protobuf.ByteString value) { descriptor; static { java.lang.String[] descriptorData = { - "\n src/main/proto/DataFormats.proto\"\262\003\n\024L" + + "\n src/main/proto/DataFormats.proto\"\301\003\n\024L" + "edgerMetadataFormat\022\022\n\nquorumSize\030\001 \002(\005\022" + "\024\n\014ensembleSize\030\002 \002(\005\022\016\n\006length\030\003 \002(\003\022\023\n" + "\013lastEntryId\030\004 \001(\003\0220\n\005state\030\005 \002(\0162\033.Ledg" + @@ -4001,18 +4058,19 @@ void setBookieId(com.google.protobuf.ByteString value) { "\006 \003(\0132\035.LedgerMetadataFormat.Segment\0224\n\n" + "digestType\030\007 \001(\0162 .LedgerMetadataFormat." + "DigestType\022\020\n\010password\030\010 \001(\014\022\025\n\rackQuoru" + - "mSize\030\t \001(\005\0327\n\007Segment\022\026\n\016ensembleMember" + - "\030\001 \003(\t\022\024\n\014firstEntryId\030\002 \002(\003\".\n\005State\022\010\n", - "\004OPEN\020\001\022\017\n\013IN_RECOVERY\020\002\022\n\n\006CLOSED\020\003\"!\n\n" + - "DigestType\022\t\n\005CRC32\020\001\022\010\n\004HMAC\020\002\"@\n\037Ledge" + - "rRereplicationLayoutFormat\022\014\n\004type\030\001 \002(\t" + - "\022\017\n\007version\030\002 \002(\005\".\n\033UnderreplicatedLedg" + - "erFormat\022\017\n\007replica\030\001 \003(\t\"^\n\014CookieForma" + - "t\022\022\n\nbookieHost\030\001 \002(\t\022\022\n\njournalDir\030\002 \002(" + - "\t\022\022\n\nledgerDirs\030\003 \002(\t\022\022\n\ninstanceId\030\004 \001(" + - "\t\"\"\n\016LockDataFormat\022\020\n\010bookieId\030\001 \001(\t\"%\n" + - "\021AuditorVoteFormat\022\020\n\010bookieId\030\001 \001(\tB\037\n\033" + - "org.apache.bookkeeper.protoH\001" + "mSize\030\t \001(\005\022\r\n\005ctime\030\n \001(\003\0327\n\007Segment\022\026\n" + + "\016ensembleMember\030\001 \003(\t\022\024\n\014firstEntryId\030\002 ", + "\002(\003\".\n\005State\022\010\n\004OPEN\020\001\022\017\n\013IN_RECOVERY\020\002\022" + + "\n\n\006CLOSED\020\003\"!\n\nDigestType\022\t\n\005CRC32\020\001\022\010\n\004" + + "HMAC\020\002\"@\n\037LedgerRereplicationLayoutForma" + + "t\022\014\n\004type\030\001 \002(\t\022\017\n\007version\030\002 \002(\005\".\n\033Unde" + + "rreplicatedLedgerFormat\022\017\n\007replica\030\001 \003(\t" + + "\"^\n\014CookieFormat\022\022\n\nbookieHost\030\001 \002(\t\022\022\n\n" + + "journalDir\030\002 \002(\t\022\022\n\nledgerDirs\030\003 \002(\t\022\022\n\n" + + "instanceId\030\004 \001(\t\"\"\n\016LockDataFormat\022\020\n\010bo" + + "okieId\030\001 \001(\t\"%\n\021AuditorVoteFormat\022\020\n\010boo" + + "kieId\030\001 \001(\tB\037\n\033org.apache.bookkeeper.pro", + "toH\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -4024,7 +4082,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( internal_static_LedgerMetadataFormat_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_LedgerMetadataFormat_descriptor, - new java.lang.String[] { "QuorumSize", "EnsembleSize", "Length", "LastEntryId", "State", "Segment", "DigestType", "Password", "AckQuorumSize", }, + new java.lang.String[] { "QuorumSize", "EnsembleSize", "Length", "LastEntryId", "State", "Segment", "DigestType", "Password", "AckQuorumSize", "Ctime", }, org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.class, org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.Builder.class); internal_static_LedgerMetadataFormat_Segment_descriptor = diff --git a/bookkeeper-server/src/main/proto/DataFormats.proto b/bookkeeper-server/src/main/proto/DataFormats.proto index 67623ac6148..6d97c3a452e 100644 --- a/bookkeeper-server/src/main/proto/DataFormats.proto +++ b/bookkeeper-server/src/main/proto/DataFormats.proto @@ -48,6 +48,8 @@ message LedgerMetadataFormat { optional bytes password = 8; optional int32 ackQuorumSize = 9; + + optional int64 ctime = 10; } message LedgerRereplicationLayoutFormat { diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java index d34c0648f9e..95b7d53667a 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java @@ -14,7 +14,6 @@ * License for the specific language governing permissions and limitations under * the License. */ - package org.apache.bookkeeper.client; import java.util.Iterator; @@ -31,25 +30,26 @@ import org.slf4j.LoggerFactory; public class ListLedgersTest extends BaseTestCase { + private final static Logger LOG = LoggerFactory.getLogger(ListLedgersTest.class); DigestType digestType; - public ListLedgersTest (DigestType digestType) { + public ListLedgersTest(DigestType digestType) { super(4); this.digestType = digestType; } - @Test(timeout=60000) + @Test(timeout = 60000) public void testListLedgers() - throws Exception { + throws Exception { int numOfLedgers = 10; ClientConfiguration conf = new ClientConfiguration() - .setZkServers(zkUtil.getZooKeeperConnectString()); + .setZkServers(zkUtil.getZooKeeperConnectString()); BookKeeper bkc = new BookKeeper(conf); - for (int i = 0; i < numOfLedgers ; i++) { + for (int i = 0; i < numOfLedgers; i++) { bkc.createLedger(digestType, "testPasswd". getBytes()).close(); } @@ -59,7 +59,7 @@ public void testListLedgers() Iterable iterable = admin.listLedgers(); int counter = 0; - for (Long lId: iterable) { + for (Long lId : iterable) { counter++; } @@ -67,11 +67,11 @@ public void testListLedgers() counter == numOfLedgers); } - @Test(timeout=60000) + @Test(timeout = 60000) public void testEmptyList() - throws Exception { + throws Exception { ClientConfiguration conf = new ClientConfiguration() - .setZkServers(zkUtil.getZooKeeperConnectString()); + .setZkServers(zkUtil.getZooKeeperConnectString()); BookKeeperAdmin admin = new BookKeeperAdmin(zkUtil. getZooKeeperConnectString()); @@ -80,16 +80,16 @@ public void testEmptyList() Assert.assertFalse("There should be no ledger", iterable.iterator().hasNext()); } - @Test(timeout=60000) + @Test(timeout = 60000) public void testRemoveNotSupported() - throws Exception { + throws Exception { int numOfLedgers = 1; ClientConfiguration conf = new ClientConfiguration() - .setZkServers(zkUtil.getZooKeeperConnectString()); + .setZkServers(zkUtil.getZooKeeperConnectString()); BookKeeper bkc = new BookKeeper(conf); - for (int i = 0; i < numOfLedgers ; i++) { + for (int i = 0; i < numOfLedgers; i++) { bkc.createLedger(digestType, "testPasswd". getBytes()).close(); } @@ -98,7 +98,7 @@ public void testRemoveNotSupported() getZooKeeperConnectString()); Iterator iterator = admin.listLedgers().iterator(); iterator.next(); - try{ + try { iterator.remove(); } catch (UnsupportedOperationException e) { // This exception is expected @@ -108,4 +108,28 @@ public void testRemoveNotSupported() Assert.fail("Remove is not supported, we shouln't have reached this point"); } + + @Test(timeout = 60000) + public void testCtimeRecorded() + throws Exception { + + ClientConfiguration conf = new ClientConfiguration() + .setZkServers(zkUtil.getZooKeeperConnectString()); + + BookKeeper bkc = new BookKeeper(conf); + + bkc.createLedger(digestType, "testPasswd". + getBytes()).close(); + + BookKeeperAdmin admin = new BookKeeperAdmin(zkUtil. + getZooKeeperConnectString()); + Iterable iterable = admin.listLedgers(); + + for (Long lId : iterable) { + LedgerHandle ledger = bkc.openLedger(lId, digestType, "testPasswd".getBytes()); + LedgerMetadata metaData = ledger.getLedgerMetadata(); + Assert.assertTrue("ctime was not recorded", metaData.getCtime() > 0); + } + + } } From c43f0b4e0a400ac87acbc278055e582177812e2d Mon Sep 17 00:00:00 2001 From: eolivelli Date: Wed, 25 Nov 2015 17:15:25 +0100 Subject: [PATCH 2/6] BOOKKEEPER-879: Record ledger creation time --- .../apache/bookkeeper/client/ListLedgersTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java index 95b7d53667a..f3464d97200 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java @@ -42,11 +42,11 @@ public ListLedgersTest(DigestType digestType) { @Test(timeout = 60000) public void testListLedgers() - throws Exception { + throws Exception { int numOfLedgers = 10; ClientConfiguration conf = new ClientConfiguration() - .setZkServers(zkUtil.getZooKeeperConnectString()); + .setZkServers(zkUtil.getZooKeeperConnectString()); BookKeeper bkc = new BookKeeper(conf); for (int i = 0; i < numOfLedgers; i++) { @@ -69,9 +69,9 @@ public void testListLedgers() @Test(timeout = 60000) public void testEmptyList() - throws Exception { + throws Exception { ClientConfiguration conf = new ClientConfiguration() - .setZkServers(zkUtil.getZooKeeperConnectString()); + .setZkServers(zkUtil.getZooKeeperConnectString()); BookKeeperAdmin admin = new BookKeeperAdmin(zkUtil. getZooKeeperConnectString()); @@ -82,11 +82,11 @@ public void testEmptyList() @Test(timeout = 60000) public void testRemoveNotSupported() - throws Exception { + throws Exception { int numOfLedgers = 1; ClientConfiguration conf = new ClientConfiguration() - .setZkServers(zkUtil.getZooKeeperConnectString()); + .setZkServers(zkUtil.getZooKeeperConnectString()); BookKeeper bkc = new BookKeeper(conf); for (int i = 0; i < numOfLedgers; i++) { @@ -108,7 +108,7 @@ public void testRemoveNotSupported() Assert.fail("Remove is not supported, we shouln't have reached this point"); } - + @Test(timeout = 60000) public void testCtimeRecorded() throws Exception { From 2a8e7008402ca7c1cf9188751b3e21ca2dcce7de Mon Sep 17 00:00:00 2001 From: eolivelli Date: Thu, 14 Jan 2016 09:02:23 +0100 Subject: [PATCH 3/6] BOOKKEEPER-879: Record ledger creation time --- .../bookkeeper/client/ListLedgersTest.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java index f3464d97200..70ff6e0ad09 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java @@ -14,6 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package org.apache.bookkeeper.client; import java.util.Iterator; @@ -30,17 +31,16 @@ import org.slf4j.LoggerFactory; public class ListLedgersTest extends BaseTestCase { - private final static Logger LOG = LoggerFactory.getLogger(ListLedgersTest.class); DigestType digestType; - public ListLedgersTest(DigestType digestType) { + public ListLedgersTest (DigestType digestType) { super(4); this.digestType = digestType; } - @Test(timeout = 60000) + @Test(timeout=60000) public void testListLedgers() throws Exception { int numOfLedgers = 10; @@ -49,7 +49,7 @@ public void testListLedgers() .setZkServers(zkUtil.getZooKeeperConnectString()); BookKeeper bkc = new BookKeeper(conf); - for (int i = 0; i < numOfLedgers; i++) { + for (int i = 0; i < numOfLedgers ; i++) { bkc.createLedger(digestType, "testPasswd". getBytes()).close(); } @@ -59,7 +59,7 @@ public void testListLedgers() Iterable iterable = admin.listLedgers(); int counter = 0; - for (Long lId : iterable) { + for (Long lId: iterable) { counter++; } @@ -67,7 +67,7 @@ public void testListLedgers() counter == numOfLedgers); } - @Test(timeout = 60000) + @Test(timeout=60000) public void testEmptyList() throws Exception { ClientConfiguration conf = new ClientConfiguration() @@ -80,7 +80,7 @@ public void testEmptyList() Assert.assertFalse("There should be no ledger", iterable.iterator().hasNext()); } - @Test(timeout = 60000) + @Test(timeout=60000) public void testRemoveNotSupported() throws Exception { int numOfLedgers = 1; @@ -89,7 +89,7 @@ public void testRemoveNotSupported() .setZkServers(zkUtil.getZooKeeperConnectString()); BookKeeper bkc = new BookKeeper(conf); - for (int i = 0; i < numOfLedgers; i++) { + for (int i = 0; i < numOfLedgers ; i++) { bkc.createLedger(digestType, "testPasswd". getBytes()).close(); } @@ -98,7 +98,7 @@ public void testRemoveNotSupported() getZooKeeperConnectString()); Iterator iterator = admin.listLedgers().iterator(); iterator.next(); - try { + try{ iterator.remove(); } catch (UnsupportedOperationException e) { // This exception is expected @@ -132,4 +132,5 @@ public void testCtimeRecorded() } } + } From 94cdee6a99fb07938cc30806bbe99d78b8bb1b30 Mon Sep 17 00:00:00 2001 From: eolivelli Date: Fri, 15 Jan 2016 16:39:44 +0100 Subject: [PATCH 4/6] BOOKKEEPER-879: Record ledger creation time --- .../org/apache/bookkeeper/client/LedgerMetadata.java | 12 +++++++++--- .../bookkeeper/meta/AbstractZkLedgerManager.java | 2 +- .../bookkeeper/meta/MSLedgerManagerFactory.java | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java index 1e153dd1519..d513c5c6bf3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java @@ -114,7 +114,7 @@ public LedgerMetadata(int ensembleSize, int writeQuorumSize, int ackQuorumSize, this.state = other.state; this.version = other.version; this.hasPassword = other.hasPassword; - this.digestType = other.digestType; + this.digestType = other.digestType; this.password = new byte[other.password.length]; System.arraycopy(other.password, 0, this.password, 0, other.password.length); // copy the ensembles @@ -319,11 +319,13 @@ private byte[] serializeVersion1() { * byte array to parse * @param version * version of the ledger metadata + * @param zkCtime + * ctime read from Zookeeper Stat, used for legacy ledgers * @return LedgerConfig * @throws IOException * if the given byte[] cannot be parsed */ - public static LedgerMetadata parseConfig(byte[] bytes, Version version) throws IOException { + public static LedgerMetadata parseConfig(byte[] bytes, Version version, Long zkCtime) throws IOException { LedgerMetadata lc = new LedgerMetadata(); lc.version = version; @@ -363,7 +365,11 @@ public static LedgerMetadata parseConfig(byte[] bytes, Version version) throws I TextFormat.merge(reader, builder); LedgerMetadataFormat data = builder.build(); lc.writeQuorumSize = data.getQuorumSize(); - lc.ctime=data.getCtime(); + if (data.hasCtime()) { + lc.ctime=data.getCtime(); + } else if (zkCtime != null) { + lc.ctime=zkCtime; + } if (data.hasAckQuorumSize()) { lc.ackQuorumSize = data.getAckQuorumSize(); } else { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java index 663650671a2..9b5cac16666 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java @@ -355,7 +355,7 @@ public void processResult(int rc, String path, Object ctx, byte[] data, Stat sta LedgerMetadata metadata; try { - metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion())); + metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()),stat.getCtime()); } catch (IOException e) { LOG.error("Could not parse ledger metadata for ledger: " + ledgerId, e); readCb.operationComplete(BKException.Code.ZKException, null); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java index 9f7ef386da3..bcd5206b80b 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java @@ -408,7 +408,7 @@ public void complete(int rc, Versioned value, Object ctx) { LedgerMetadata metadata; try { metadata = LedgerMetadata - .parseConfig(value.getValue().getField(META_FIELD), value.getVersion()); + .parseConfig(value.getValue().getField(META_FIELD), value.getVersion(),null); } catch (IOException e) { LOG.error("Could not parse ledger metadata for ledger " + ledgerId + " : ", e); readCb.operationComplete(BKException.Code.MetaStoreException, null); From 227d50c7db634a23bd30cc3478ddfe3c307aaa62 Mon Sep 17 00:00:00 2001 From: eolivelli Date: Sat, 30 Jan 2016 22:51:43 +0100 Subject: [PATCH 5/6] BOOKKEEPER-879: Record ledger creation time --- .../java/org/apache/bookkeeper/client/LedgerMetadata.java | 4 ++-- .../apache/bookkeeper/meta/AbstractZkLedgerManager.java | 8 ++++++-- .../apache/bookkeeper/meta/MSLedgerManagerFactory.java | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java index d513c5c6bf3..3f4d8d73d7f 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java @@ -366,9 +366,9 @@ public static LedgerMetadata parseConfig(byte[] bytes, Version version, Long zkC LedgerMetadataFormat data = builder.build(); lc.writeQuorumSize = data.getQuorumSize(); if (data.hasCtime()) { - lc.ctime=data.getCtime(); + lc.ctime = data.getCtime(); } else if (zkCtime != null) { - lc.ctime=zkCtime; + lc.ctime = zkCtime; } if (data.hasAckQuorumSize()) { lc.ackQuorumSize = data.getAckQuorumSize(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java index 9b5cac16666..e52dd6338b0 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java @@ -352,10 +352,14 @@ public void processResult(int rc, String path, Object ctx, byte[] data, Stat sta readCb.operationComplete(BKException.Code.ZKException, null); return; } - + if (stat == null) { + LOG.error("Could not parse ledger metadata for ledger: " + ledgerId+". Stat object is null"); + readCb.operationComplete(BKException.Code.ZKException, null); + return; + } LedgerMetadata metadata; try { - metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()),stat.getCtime()); + metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()), stat.getCtime()); } catch (IOException e) { LOG.error("Could not parse ledger metadata for ledger: " + ledgerId, e); readCb.operationComplete(BKException.Code.ZKException, null); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java index bcd5206b80b..05b1b870e59 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java @@ -408,7 +408,7 @@ public void complete(int rc, Versioned value, Object ctx) { LedgerMetadata metadata; try { metadata = LedgerMetadata - .parseConfig(value.getValue().getField(META_FIELD), value.getVersion(),null); + .parseConfig(value.getValue().getField(META_FIELD), value.getVersion(), null); } catch (IOException e) { LOG.error("Could not parse ledger metadata for ledger " + ledgerId + " : ", e); readCb.operationComplete(BKException.Code.MetaStoreException, null); From 509ad4f31c5f10f96a0272d842120984ce2db04b Mon Sep 17 00:00:00 2001 From: eolivelli Date: Wed, 3 Feb 2016 08:40:09 +0100 Subject: [PATCH 6/6] BOOKKEEPER-879: Record ledger creation time --- .../org/apache/bookkeeper/client/LedgerMetadata.java | 11 ++++++----- .../bookkeeper/meta/AbstractZkLedgerManager.java | 3 ++- .../bookkeeper/meta/MSLedgerManagerFactory.java | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java index 3f4d8d73d7f..a58adbaa5ba 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java @@ -37,6 +37,7 @@ import java.util.TreeMap; import static com.google.common.base.Charsets.UTF_8; +import com.google.common.base.Optional; /** * This class encapsulates all the ledger metadata that is persistently stored @@ -319,13 +320,13 @@ private byte[] serializeVersion1() { * byte array to parse * @param version * version of the ledger metadata - * @param zkCtime - * ctime read from Zookeeper Stat, used for legacy ledgers + * @param msCtime + * metadata store creation time, used for legacy ledgers * @return LedgerConfig * @throws IOException * if the given byte[] cannot be parsed */ - public static LedgerMetadata parseConfig(byte[] bytes, Version version, Long zkCtime) throws IOException { + public static LedgerMetadata parseConfig(byte[] bytes, Version version, Optional msCtime) throws IOException { LedgerMetadata lc = new LedgerMetadata(); lc.version = version; @@ -367,8 +368,8 @@ public static LedgerMetadata parseConfig(byte[] bytes, Version version, Long zkC lc.writeQuorumSize = data.getQuorumSize(); if (data.hasCtime()) { lc.ctime = data.getCtime(); - } else if (zkCtime != null) { - lc.ctime = zkCtime; + } else if (msCtime.isPresent()) { + lc.ctime = msCtime.get(); } if (data.hasAckQuorumSize()) { lc.ackQuorumSize = data.getAckQuorumSize(); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java index e52dd6338b0..decf7bacfba 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java @@ -17,6 +17,7 @@ */ package org.apache.bookkeeper.meta; +import com.google.common.base.Optional; import java.io.IOException; import java.util.HashSet; import java.util.List; @@ -359,7 +360,7 @@ public void processResult(int rc, String path, Object ctx, byte[] data, Stat sta } LedgerMetadata metadata; try { - metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()), stat.getCtime()); + metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()), Optional.of(stat.getCtime())); } catch (IOException e) { LOG.error("Could not parse ledger metadata for ledger: " + ledgerId, e); readCb.operationComplete(BKException.Code.ZKException, null); diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java index 05b1b870e59..8a82d41b891 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java @@ -17,6 +17,7 @@ */ package org.apache.bookkeeper.meta; +import com.google.common.base.Optional; import static org.apache.bookkeeper.metastore.MetastoreTable.ALL_FIELDS; import static org.apache.bookkeeper.metastore.MetastoreTable.NON_FIELDS; @@ -408,7 +409,7 @@ public void complete(int rc, Versioned value, Object ctx) { LedgerMetadata metadata; try { metadata = LedgerMetadata - .parseConfig(value.getValue().getField(META_FIELD), value.getVersion(), null); + .parseConfig(value.getValue().getField(META_FIELD), value.getVersion(), Optional.absent()); } catch (IOException e) { LOG.error("Could not parse ledger metadata for ledger " + ledgerId + " : ", e); readCb.operationComplete(BKException.Code.MetaStoreException, null);