From 83cf844cf5b4499a722f114d05c12e7440fdc64b Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Tue, 12 Mar 2019 16:59:03 +0800 Subject: [PATCH 1/5] Migrate command `ledger` --- .../apache/bookkeeper/bookie/BookieShell.java | 96 +-------- .../cli/commands/bookie/LedgerCommand.java | 194 ++++++++++++++++++ .../cli/commands/BookieCommandGroup.java | 2 + .../commands/bookie/LedgerCommandTest.java | 166 +++++++++++++++ 4 files changed, 367 insertions(+), 91 deletions(-) create mode 100644 bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java create mode 100644 tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommandTest.java diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java index 628690e6969..f2bfed1936a 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java @@ -111,6 +111,7 @@ import org.apache.bookkeeper.stats.NullStatsLogger; import org.apache.bookkeeper.tools.cli.commands.bookie.FormatCommand; import org.apache.bookkeeper.tools.cli.commands.bookie.LastMarkCommand; +import org.apache.bookkeeper.tools.cli.commands.bookie.LedgerCommand; import org.apache.bookkeeper.tools.cli.commands.bookies.InfoCommand; import org.apache.bookkeeper.tools.cli.commands.bookies.ListBookiesCommand; import org.apache.bookkeeper.tools.cli.commands.client.SimpleTestCommand; @@ -143,7 +144,6 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.mutable.MutableBoolean; -import org.apache.commons.lang.mutable.MutableLong; import org.apache.commons.lang3.ArrayUtils; import org.apache.zookeeper.AsyncCallback; import org.apache.zookeeper.AsyncCallback.VoidCallback; @@ -715,96 +715,10 @@ class LedgerCmd extends MyCommand { @Override public int runCmd(CommandLine cmdLine) throws Exception { - String[] leftArgs = cmdLine.getArgs(); - if (leftArgs.length <= 0) { - printErrorLine("ERROR: missing ledger id"); - printUsage(); - return -1; - } - - boolean printMeta = false; - if (cmdLine.hasOption("m")) { - printMeta = true; - } - long ledgerId; - try { - ledgerId = ledgerIdFormatter.readLedgerId(leftArgs[0]); - } catch (IllegalArgumentException iae) { - printErrorLine("ERROR: invalid ledger id " + leftArgs[0]); - printUsage(); - return -1; - } - - if (bkConf.getLedgerStorageClass().equals(DbLedgerStorage.class.getName())) { - // dump ledger info - try { - DbLedgerStorage.readLedgerIndexEntries(ledgerId, bkConf, - (currentEntry, entryLogId, position) -> printInfoLine( - "entry " + currentEntry + "\t:\t(log: " + entryLogId + ", pos: " + position + ")")); - } catch (IOException e) { - System.err.printf("ERROR: initializing dbLedgerStorage %s", e.getMessage()); - return -1; - } - } else if ((bkConf.getLedgerStorageClass().equals(SortedLedgerStorage.class.getName()) - || bkConf.getLedgerStorageClass().equals(InterleavedLedgerStorage.class.getName()))) { - ServerConfiguration conf = new ServerConfiguration(bkConf); - InterleavedLedgerStorage interleavedStorage = new InterleavedLedgerStorage(); - Bookie.mountLedgerStorageOffline(conf, interleavedStorage); - - if (printMeta) { - // print meta - printInfoLine("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); - LedgerCache.LedgerIndexMetadata meta = interleavedStorage.readLedgerIndexMetadata(ledgerId); - printInfoLine("master key : " + meta.getMasterKeyHex()); - - long size = meta.size; - if (size % 8 == 0) { - printInfoLine("size : " + size); - } else { - printInfoLine("size : " + size - + " (not aligned with 8, may be corrupted or under flushing now)"); - } - - printInfoLine("entries : " + (size / 8)); - printInfoLine("isFenced : " + meta.fenced); - } - - try { - // dump ledger info - printInfoLine("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); - for (LedgerCache.PageEntries page : interleavedStorage.getIndexEntries(ledgerId)) { - final MutableLong curEntry = new MutableLong(page.getFirstEntry()); - try (LedgerEntryPage lep = page.getLEP()){ - lep.getEntries((entry, offset) -> { - while (curEntry.longValue() < entry) { - printInfoLine("entry " + curEntry + "\t:\tN/A"); - curEntry.increment(); - } - long entryLogId = offset >> 32L; - long pos = offset & 0xffffffffL; - printInfoLine("entry " + curEntry + "\t:\t(log:" + entryLogId + ", pos: " + pos + ")"); - curEntry.increment(); - return true; - }); - } catch (IOException ie) { - printInfoLine("Failed to read index page @ " + page.getFirstEntry() - + ", the index file may be corrupted : " - + ie.getMessage()); - return 1; - } - - while (curEntry.longValue() < page.getLastEntry()) { - printInfoLine("entry " + curEntry + "\t:\tN/A"); - curEntry.increment(); - } - } - } catch (IOException ie) { - LOG.error("Failed to read index page"); - return 1; - } - } - - return 0; + LedgerCommand cmd = new LedgerCommand(ledgerIdFormatter); + LedgerCommand.LedgerFlags flags = new LedgerCommand.LedgerFlags(); + cmd.apply(bkConf, flags); + return 1; } @Override diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java new file mode 100644 index 00000000000..7b27783571c --- /dev/null +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.tools.cli.commands.bookie; + +import com.beust.jcommander.Parameter; +import com.google.common.util.concurrent.UncheckedExecutionException; +import java.io.IOException; +import lombok.Setter; +import lombok.experimental.Accessors; +import org.apache.bookkeeper.bookie.Bookie; +import org.apache.bookkeeper.bookie.InterleavedLedgerStorage; +import org.apache.bookkeeper.bookie.LedgerCache; +import org.apache.bookkeeper.bookie.LedgerEntryPage; +import org.apache.bookkeeper.bookie.SortedLedgerStorage; +import org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.tools.cli.helpers.BookieCommand; +import org.apache.bookkeeper.tools.framework.CliFlags; +import org.apache.bookkeeper.tools.framework.CliSpec; +import org.apache.bookkeeper.util.LedgerIdFormatter; +import org.apache.commons.lang.mutable.MutableLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Command to dump ledger index entries into readable format. + */ +public class LedgerCommand extends BookieCommand { + + static final Logger LOG = LoggerFactory.getLogger(LedgerCommand.class); + + private static final String NAME = "ledger"; + private static final String DESC = "Dump ledger index entries into readable format"; + + private LedgerIdFormatter ledgerIdFormatter; + + public LedgerCommand() { + this(new LedgerFlags()); + } + + public LedgerCommand(LedgerIdFormatter ledgerIdFormatter) { + this(new LedgerFlags()); + this.ledgerIdFormatter = ledgerIdFormatter; + } + + private LedgerCommand(LedgerFlags flags) { + super(CliSpec.newBuilder().withName(NAME).withDescription(DESC).withFlags(flags).build()); + } + + /** + * Flags for ledger command. + */ + @Accessors(fluent = true) + @Setter + public static class LedgerFlags extends CliFlags { + + @Parameter(names = { "-id", "--ledgerId" }, description = "Specific ledger id", required = true) + private long ledgerId; + + @Parameter(names = { "-m", "--meta" }, description = "Print meta information") + private boolean meta; + + @Parameter(names = { "-l", "--ledgeridformatter" }, description = "Set ledger id formatter") + private String ledgerIdFormatter = ""; + } + + @Override + public boolean apply(ServerConfiguration conf, LedgerFlags cmdFlags) { + initLedgerIdFormatter(conf, cmdFlags); + long ledgerId = cmdFlags.ledgerId; + if (conf.getLedgerStorageClass().equals(DbLedgerStorage.class.getName())) { + // dump ledger info + if (!dumpLedgerInfo(ledgerId, conf)) { + return false; + } + } else if (conf.getLedgerStorageClass().equals(SortedLedgerStorage.class.getName()) || conf + .getLedgerStorageClass().equals(InterleavedLedgerStorage.class.getName())) { + ServerConfiguration tConf = new ServerConfiguration(conf); + InterleavedLedgerStorage interleavedLedgerStorage = new InterleavedLedgerStorage(); + try { + Bookie.mountLedgerStorageOffline(tConf, interleavedLedgerStorage); + } catch (IOException e) { + throw new UncheckedExecutionException(e.getMessage(), e); + } + + if (cmdFlags.meta) { + // print meta + printMeta(ledgerId, interleavedLedgerStorage); + } + + try { + printInfoLine("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); + for (LedgerCache.PageEntries page : interleavedLedgerStorage.getIndexEntries(ledgerId)) { + if (printPageEntries(page)) { + return true; + } + } + } catch (IOException e) { + LOG.error("Failed to read index page"); + return true; + } + + } + return true; + } + + private void initLedgerIdFormatter(ServerConfiguration conf, LedgerFlags flags) { + if (flags.ledgerIdFormatter.equals("")) { + this.ledgerIdFormatter = LedgerIdFormatter.newLedgerIdFormatter(conf); + } else { + this.ledgerIdFormatter = LedgerIdFormatter.newLedgerIdFormatter(flags.ledgerIdFormatter, conf); + } + } + + private boolean dumpLedgerInfo(long ledgerId, ServerConfiguration conf) { + try { + DbLedgerStorage.readLedgerIndexEntries(ledgerId, conf, (currentEntry, entryLodId, position) -> System.out + .println("entry " + currentEntry + "\t:\t(log: " + entryLodId + ", pos: " + position + ")")); + } catch (IOException e) { + System.err.printf("ERROR: initializing dbLedgerStorage %s", e.getMessage()); + return false; + } + return true; + } + + private void printMeta(long ledgerId, InterleavedLedgerStorage interleavedLedgerStorage) { + printInfoLine("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); + try { + LedgerCache.LedgerIndexMetadata meta = interleavedLedgerStorage.readLedgerIndexMetadata(ledgerId); + printInfoLine("master key : " + meta.getMasterKeyHex()); + long size = meta.size; + if (size % 8 == 0) { + printInfoLine("size : " + size); + } else { + printInfoLine("size : " + size + "(not aligned with 8, may be corrupted or under flushing now)"); + } + + printInfoLine("entries : " + (size / 8)); + printInfoLine("isFenced : " + meta.fenced); + + } catch (IOException e) { + throw new UncheckedExecutionException(e.getMessage(), e); + } + } + + private boolean printPageEntries(LedgerCache.PageEntries page) { + final MutableLong curEntry = new MutableLong(page.getFirstEntry()); + try (LedgerEntryPage lep = page.getLEP()) { + lep.getEntries((entry, offset) -> { + while (curEntry.longValue() < entry) { + printInfoLine("entry " + curEntry + "\t:\tN/A"); + curEntry.increment(); + } + long entryLogId = offset >> 32L; + long pos = offset & 0xffffffffL; + printInfoLine("entry " + curEntry + "\t:\t(log:" + entryLogId + ", pos: " + pos + ")"); + curEntry.increment(); + return true; + }); + } catch (Exception e) { + printInfoLine("Failed to read index page @ " + page.getFirstEntry() + + ", the index file may be corrupted : " + e.getMessage()); + return true; + } + + while (curEntry.longValue() < page.getLastEntry()) { + printInfoLine("entry " + curEntry + "\t:\tN/A"); + curEntry.increment(); + } + + return false; + } + + + private void printInfoLine(String mes) { + System.out.println(mes); + } +} diff --git a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java index cfa37e37e1d..1a57967a417 100644 --- a/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java +++ b/tools/ledger/src/main/java/org/apache/bookkeeper/tools/cli/commands/BookieCommandGroup.java @@ -23,6 +23,7 @@ import org.apache.bookkeeper.tools.cli.BKCtl; import org.apache.bookkeeper.tools.cli.commands.bookie.FormatCommand; import org.apache.bookkeeper.tools.cli.commands.bookie.LastMarkCommand; +import org.apache.bookkeeper.tools.cli.commands.bookie.LedgerCommand; import org.apache.bookkeeper.tools.common.BKFlags; import org.apache.bookkeeper.tools.framework.CliCommandGroup; import org.apache.bookkeeper.tools.framework.CliSpec; @@ -42,6 +43,7 @@ public class BookieCommandGroup extends CliCommandGroup { .withCategory(CATEGORY_INFRA_SERVICE) .addCommand(new LastMarkCommand()) .addCommand(new FormatCommand()) + .addCommand(new LedgerCommand()) .build(); public BookieCommandGroup() { diff --git a/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommandTest.java b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommandTest.java new file mode 100644 index 00000000000..1023458fef0 --- /dev/null +++ b/tools/ledger/src/test/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommandTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.bookkeeper.tools.cli.commands.bookie; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.internal.verification.VerificationModeFactory.times; + +import java.util.Iterator; +import org.apache.bookkeeper.bookie.Bookie; +import org.apache.bookkeeper.bookie.InterleavedLedgerStorage; +import org.apache.bookkeeper.bookie.LedgerCache; +import org.apache.bookkeeper.bookie.LedgerEntryPage; +import org.apache.bookkeeper.bookie.LedgerStorage; +import org.apache.bookkeeper.bookie.SortedLedgerStorage; +import org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage; +import org.apache.bookkeeper.bookie.storage.ldb.SingleDirectoryDbLedgerStorage; +import org.apache.bookkeeper.conf.ServerConfiguration; +import org.apache.bookkeeper.tools.cli.helpers.BookieCommandTestBase; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +/** + * Unit test for {@link LedgerCommand}. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest({ DbLedgerStorage.class, SortedLedgerStorage.class, InterleavedLedgerStorage.class, Bookie.class, + LedgerStorage.class, LedgerCache.PageEntries.class, LedgerCache.PageEntriesIterable.class, LedgerCommand.class, + LedgerCache.LedgerIndexMetadata.class }) +public class LedgerCommandTest extends BookieCommandTestBase { + + private LedgerCache.LedgerIndexMetadata metadata; + private ServerConfiguration tConf; + + public LedgerCommandTest() { + super(3, 0); + } + + public void setup() throws Exception { + super.setup(); + PowerMockito.whenNew(ServerConfiguration.class).withNoArguments().thenReturn(conf); + + PowerMockito.mockStatic(DbLedgerStorage.class); + PowerMockito.doAnswer(invocationOnMock -> { + SingleDirectoryDbLedgerStorage.LedgerLoggerProcessor p = invocationOnMock.getArgument(2); + p.process(1L, 1L, 1L); + return true; + }).when(DbLedgerStorage.class, "readLedgerIndexEntries", anyLong(), any(ServerConfiguration.class), + any(SingleDirectoryDbLedgerStorage.LedgerLoggerProcessor.class)); + PowerMockito.when(DbLedgerStorage.class.getName()) + .thenReturn("org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage"); + + tConf = PowerMockito.mock(ServerConfiguration.class); + PowerMockito.whenNew(ServerConfiguration.class).withArguments(ServerConfiguration.class) + .thenReturn(tConf); + + InterleavedLedgerStorage interleavedLedgerStorage = PowerMockito.mock(InterleavedLedgerStorage.class); + PowerMockito.whenNew(InterleavedLedgerStorage.class).withNoArguments().thenReturn(interleavedLedgerStorage); + + PowerMockito.mockStatic(Bookie.class); + PowerMockito.when(Bookie.mountLedgerStorageOffline(eq(tConf), eq(interleavedLedgerStorage))) + .thenReturn(PowerMockito.mock(LedgerStorage.class)); + + LedgerCache.PageEntries e = PowerMockito.mock(LedgerCache.PageEntries.class); + LedgerCache.PageEntriesIterable i = PowerMockito.mock(LedgerCache.PageEntriesIterable.class); + PowerMockito.when(interleavedLedgerStorage.getIndexEntries(anyLong())).thenReturn(i); + PowerMockito.when(i.iterator()).thenReturn(getPageIterator(e)); + LedgerEntryPage lep = PowerMockito.mock(LedgerEntryPage.class); + PowerMockito.when(e.getLEP()).thenReturn(lep); + + metadata = PowerMockito.mock(LedgerCache.LedgerIndexMetadata.class); + PowerMockito.when(interleavedLedgerStorage.readLedgerIndexMetadata(anyLong())).thenReturn(metadata); + PowerMockito.when(metadata.getMasterKeyHex()).thenReturn(""); + } + + public Iterator getPageIterator(LedgerCache.PageEntries page) { + Iterator i = new Iterator() { + int i = 0; + + @Override + public boolean hasNext() { + if (i < 2) { + i++; + return true; + } + return false; + } + + @Override + public LedgerCache.PageEntries next() { + return page; + } + }; + return i; + } + + // Test without ledger id + @Test + public void testWithoutLedgerId() { + testLedgerCommand(""); + } + + // test ledger command without args + @Test + public void testNoArguments() { + testLedgerCommand("-id", "1"); + } + + @Test + public void testWithMeta() throws Exception { + LedgerCommand cmd = new LedgerCommand(); + cmd.apply(bkFlags, new String[] { "-id", "1", "-m" }); + + PowerMockito.verifyNew(ServerConfiguration.class, times(1)).withArguments(eq(conf)); + PowerMockito.verifyNew(InterleavedLedgerStorage.class, times(1)).withNoArguments(); + + verify(metadata, times(1)).getMasterKeyHex(); + } + + @Test + public void testDbLedgerStorage() throws Exception { + conf.setLedgerStorageClass("org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage"); + PowerMockito.whenNew(ServerConfiguration.class).withNoArguments().thenReturn(conf); + LedgerCommand cmd = new LedgerCommand(); + Assert.assertTrue(cmd.apply(bkFlags, new String[]{"-id", "1"})); + } + + // test update formatter by flag + @Test + public void testFormatterFlag() { + testLedgerCommand("-id", "1", "-l", "hex"); + } + + public void testLedgerCommand(String... args) { + LedgerCommand ledgerCommand = new LedgerCommand(); + try { + ledgerCommand.apply(bkFlags, args); + } catch (IllegalArgumentException iae) { + if (!iae.getMessage().equals("No ledger id is specified")) { + Assert.fail("exception is not expect ! "); + } + } + } +} From f5c12ef51ca4ff29e964f61617fb88209714526d Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Tue, 12 Mar 2019 17:16:49 +0800 Subject: [PATCH 2/5] Fix ci problem, make static function public --- .../src/main/java/org/apache/bookkeeper/bookie/Bookie.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java index 16b63121a9c..bf5809c6bc6 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java @@ -625,9 +625,7 @@ private static LedgerStorage buildLedgerStorage(ServerConfiguration conf) throws * @return Passed ledgerStorage instance * @throws IOException */ - static LedgerStorage mountLedgerStorageOffline( - ServerConfiguration conf, - LedgerStorage ledgerStorage) throws IOException { + public static LedgerStorage mountLedgerStorageOffline(ServerConfiguration conf, LedgerStorage ledgerStorage) throws IOException { StatsLogger statsLogger = NullStatsLogger.INSTANCE; DiskChecker diskChecker = new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()); From 996219192561dfccb62999459c0ab1928c914486 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Tue, 12 Mar 2019 17:32:18 +0800 Subject: [PATCH 3/5] Keep one line in 120 characters --- .../src/main/java/org/apache/bookkeeper/bookie/Bookie.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java index bf5809c6bc6..44c950978de 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java @@ -625,7 +625,8 @@ private static LedgerStorage buildLedgerStorage(ServerConfiguration conf) throws * @return Passed ledgerStorage instance * @throws IOException */ - public static LedgerStorage mountLedgerStorageOffline(ServerConfiguration conf, LedgerStorage ledgerStorage) throws IOException { + public static LedgerStorage mountLedgerStorageOffline(ServerConfiguration conf, LedgerStorage ledgerStorage) + throws IOException { StatsLogger statsLogger = NullStatsLogger.INSTANCE; DiskChecker diskChecker = new DiskChecker(conf.getDiskUsageThreshold(), conf.getDiskUsageWarnThreshold()); From 3be23e7f3c0e6049d2bbcd1a6d186505c9a99fc1 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Tue, 12 Mar 2019 19:54:07 +0800 Subject: [PATCH 4/5] Fix compatibility --- .../apache/bookkeeper/bookie/BookieShell.java | 12 ++++-- .../cli/commands/bookie/LedgerCommand.java | 37 ++++++++++++------- .../bookie/InterleavedLedgerStorageTest.java | 3 +- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java index f2bfed1936a..b659e670666 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java @@ -240,8 +240,9 @@ public interface Command { void printUsage(); } - void printInfoLine(String s) { + Boolean printInfoLine(String s) { System.out.println(s); + return true; } void printErrorLine(String s) { @@ -716,9 +717,14 @@ class LedgerCmd extends MyCommand { @Override public int runCmd(CommandLine cmdLine) throws Exception { LedgerCommand cmd = new LedgerCommand(ledgerIdFormatter); + cmd.setPrint(BookieShell.this::printInfoLine); LedgerCommand.LedgerFlags flags = new LedgerCommand.LedgerFlags(); - cmd.apply(bkConf, flags); - return 1; + if (cmdLine.hasOption("m")) { + flags.meta(true); + } + flags.ledgerId(Long.parseLong(cmdLine.getArgs()[0])); + boolean result = cmd.apply(bkConf, flags); + return (result) ? 0 : 1; } @Override diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java index 7b27783571c..6303af3b94b 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java @@ -21,6 +21,8 @@ import com.beust.jcommander.Parameter; import com.google.common.util.concurrent.UncheckedExecutionException; import java.io.IOException; +import java.util.function.Function; + import lombok.Setter; import lombok.experimental.Accessors; import org.apache.bookkeeper.bookie.Bookie; @@ -50,6 +52,12 @@ public class LedgerCommand extends BookieCommand { private LedgerIdFormatter ledgerIdFormatter; + private Function print = this::printInfoLine; + + public void setPrint(Function print) { + this.print = print; + } + public LedgerCommand() { this(new LedgerFlags()); } @@ -105,7 +113,7 @@ public boolean apply(ServerConfiguration conf, LedgerFlags cmdFlags) { } try { - printInfoLine("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); + print.apply("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); for (LedgerCache.PageEntries page : interleavedLedgerStorage.getIndexEntries(ledgerId)) { if (printPageEntries(page)) { return true; @@ -140,20 +148,19 @@ private boolean dumpLedgerInfo(long ledgerId, ServerConfiguration conf) { } private void printMeta(long ledgerId, InterleavedLedgerStorage interleavedLedgerStorage) { - printInfoLine("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); + print.apply("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); try { LedgerCache.LedgerIndexMetadata meta = interleavedLedgerStorage.readLedgerIndexMetadata(ledgerId); - printInfoLine("master key : " + meta.getMasterKeyHex()); + print.apply("master key : " + meta.getMasterKeyHex()); long size = meta.size; if (size % 8 == 0) { - printInfoLine("size : " + size); + print.apply("size : " + size); } else { - printInfoLine("size : " + size + "(not aligned with 8, may be corrupted or under flushing now)"); + print.apply("size : " + size + "(not aligned with 8, may be corrupted or under flushing now)"); } - printInfoLine("entries : " + (size / 8)); - printInfoLine("isFenced : " + meta.fenced); - + print.apply("entries : " + (size / 8)); + print.apply("isFenced : " + meta.fenced); } catch (IOException e) { throw new UncheckedExecutionException(e.getMessage(), e); } @@ -164,23 +171,24 @@ private boolean printPageEntries(LedgerCache.PageEntries page) { try (LedgerEntryPage lep = page.getLEP()) { lep.getEntries((entry, offset) -> { while (curEntry.longValue() < entry) { - printInfoLine("entry " + curEntry + "\t:\tN/A"); + print.apply("entry " + curEntry + "\t:\tN/A"); curEntry.increment(); } long entryLogId = offset >> 32L; long pos = offset & 0xffffffffL; - printInfoLine("entry " + curEntry + "\t:\t(log:" + entryLogId + ", pos: " + pos + ")"); + print.apply("entry " + curEntry + "\t:\t(log:" + entryLogId + ", pos: " + pos + ")"); curEntry.increment(); return true; }); } catch (Exception e) { - printInfoLine("Failed to read index page @ " + page.getFirstEntry() - + ", the index file may be corrupted : " + e.getMessage()); + print.apply( + "Failed to read index page @ " + page.getFirstEntry() + ", the index file may be corrupted : " + e + .getMessage()); return true; } while (curEntry.longValue() < page.getLastEntry()) { - printInfoLine("entry " + curEntry + "\t:\tN/A"); + print.apply("entry " + curEntry + "\t:\tN/A"); curEntry.increment(); } @@ -188,7 +196,8 @@ private boolean printPageEntries(LedgerCache.PageEntries page) { } - private void printInfoLine(String mes) { + private Boolean printInfoLine(String mes) { System.out.println(mes); + return true; } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorageTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorageTest.java index c12ed9126e5..e54a135a9e4 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorageTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorageTest.java @@ -350,7 +350,7 @@ void validate(long foundEntries) { BookieShell shell = new BookieShell( LedgerIdFormatter.LONG_LEDGERID_FORMATTER, EntryFormatter.STRING_FORMATTER) { @Override - void printInfoLine(String s) { + Boolean printInfoLine(String s) { Matcher matcher = entryPattern.matcher(s); System.out.println(s); if (matcher.matches()) { @@ -375,6 +375,7 @@ void printInfoLine(String s) { } else { foundMetadata.check(s); } + return true; } }; shell.setConf(conf); From 1acedd39869100823ec3198320cf76013289ca55 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Wed, 13 Mar 2019 10:00:58 +0800 Subject: [PATCH 5/5] Replace function type with consumer --- .../apache/bookkeeper/bookie/BookieShell.java | 3 +- .../cli/commands/bookie/LedgerCommand.java | 35 +++++++++---------- .../bookie/InterleavedLedgerStorageTest.java | 3 +- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java index b659e670666..28f864dc0f3 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java @@ -240,9 +240,8 @@ public interface Command { void printUsage(); } - Boolean printInfoLine(String s) { + void printInfoLine(String s) { System.out.println(s); - return true; } void printErrorLine(String s) { diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java index 6303af3b94b..faaefcb82a9 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tools/cli/commands/bookie/LedgerCommand.java @@ -21,7 +21,7 @@ import com.beust.jcommander.Parameter; import com.google.common.util.concurrent.UncheckedExecutionException; import java.io.IOException; -import java.util.function.Function; +import java.util.function.Consumer; import lombok.Setter; import lombok.experimental.Accessors; @@ -52,9 +52,9 @@ public class LedgerCommand extends BookieCommand { private LedgerIdFormatter ledgerIdFormatter; - private Function print = this::printInfoLine; + private Consumer print = this::printInfoLine; - public void setPrint(Function print) { + public void setPrint(Consumer print) { this.print = print; } @@ -97,8 +97,8 @@ public boolean apply(ServerConfiguration conf, LedgerFlags cmdFlags) { if (!dumpLedgerInfo(ledgerId, conf)) { return false; } - } else if (conf.getLedgerStorageClass().equals(SortedLedgerStorage.class.getName()) || conf - .getLedgerStorageClass().equals(InterleavedLedgerStorage.class.getName())) { + } else if (conf.getLedgerStorageClass().equals(SortedLedgerStorage.class.getName()) + || conf.getLedgerStorageClass().equals(InterleavedLedgerStorage.class.getName())) { ServerConfiguration tConf = new ServerConfiguration(conf); InterleavedLedgerStorage interleavedLedgerStorage = new InterleavedLedgerStorage(); try { @@ -113,7 +113,7 @@ public boolean apply(ServerConfiguration conf, LedgerFlags cmdFlags) { } try { - print.apply("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); + print.accept("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); for (LedgerCache.PageEntries page : interleavedLedgerStorage.getIndexEntries(ledgerId)) { if (printPageEntries(page)) { return true; @@ -148,19 +148,19 @@ private boolean dumpLedgerInfo(long ledgerId, ServerConfiguration conf) { } private void printMeta(long ledgerId, InterleavedLedgerStorage interleavedLedgerStorage) { - print.apply("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); + print.accept("===== LEDGER: " + ledgerIdFormatter.formatLedgerId(ledgerId) + " ====="); try { LedgerCache.LedgerIndexMetadata meta = interleavedLedgerStorage.readLedgerIndexMetadata(ledgerId); - print.apply("master key : " + meta.getMasterKeyHex()); + print.accept("master key : " + meta.getMasterKeyHex()); long size = meta.size; if (size % 8 == 0) { - print.apply("size : " + size); + print.accept("size : " + size); } else { - print.apply("size : " + size + "(not aligned with 8, may be corrupted or under flushing now)"); + print.accept("size : " + size + "(not aligned with 8, may be corrupted or under flushing now)"); } - print.apply("entries : " + (size / 8)); - print.apply("isFenced : " + meta.fenced); + print.accept("entries : " + (size / 8)); + print.accept("isFenced : " + meta.fenced); } catch (IOException e) { throw new UncheckedExecutionException(e.getMessage(), e); } @@ -171,24 +171,24 @@ private boolean printPageEntries(LedgerCache.PageEntries page) { try (LedgerEntryPage lep = page.getLEP()) { lep.getEntries((entry, offset) -> { while (curEntry.longValue() < entry) { - print.apply("entry " + curEntry + "\t:\tN/A"); + print.accept("entry " + curEntry + "\t:\tN/A"); curEntry.increment(); } long entryLogId = offset >> 32L; long pos = offset & 0xffffffffL; - print.apply("entry " + curEntry + "\t:\t(log:" + entryLogId + ", pos: " + pos + ")"); + print.accept("entry " + curEntry + "\t:\t(log:" + entryLogId + ", pos: " + pos + ")"); curEntry.increment(); return true; }); } catch (Exception e) { - print.apply( + print.accept( "Failed to read index page @ " + page.getFirstEntry() + ", the index file may be corrupted : " + e .getMessage()); return true; } while (curEntry.longValue() < page.getLastEntry()) { - print.apply("entry " + curEntry + "\t:\tN/A"); + print.accept("entry " + curEntry + "\t:\tN/A"); curEntry.increment(); } @@ -196,8 +196,7 @@ private boolean printPageEntries(LedgerCache.PageEntries page) { } - private Boolean printInfoLine(String mes) { + private void printInfoLine(String mes) { System.out.println(mes); - return true; } } diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorageTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorageTest.java index e54a135a9e4..c12ed9126e5 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorageTest.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/InterleavedLedgerStorageTest.java @@ -350,7 +350,7 @@ void validate(long foundEntries) { BookieShell shell = new BookieShell( LedgerIdFormatter.LONG_LEDGERID_FORMATTER, EntryFormatter.STRING_FORMATTER) { @Override - Boolean printInfoLine(String s) { + void printInfoLine(String s) { Matcher matcher = entryPattern.matcher(s); System.out.println(s); if (matcher.matches()) { @@ -375,7 +375,6 @@ Boolean printInfoLine(String s) { } else { foundMetadata.check(s); } - return true; } }; shell.setConf(conf);