diff --git a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java index 776a3d9f3..c97ba72e3 100644 --- a/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java +++ b/priam/src/main/java/com/netflix/priam/backup/BackupRestoreUtil.java @@ -30,17 +30,13 @@ import javax.inject.Provider; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -/** Created by aagrawal on 8/14/17. */ +/** Helper methods applicable to both backup and restore */ public class BackupRestoreUtil { - private static final Logger logger = LoggerFactory.getLogger(BackupRestoreUtil.class); private static final Pattern columnFamilyFilterPattern = Pattern.compile(".\\.."); - private Map> includeFilter; - private Map> excludeFilter; + private final Map> includeFilter; + private final Map> excludeFilter; - public static final List FILTER_KEYSPACE = Collections.singletonList("OpsCenter"); private static final Map> FILTER_COLUMN_FAMILY = ImmutableMap.of( "system", @@ -49,43 +45,26 @@ public class BackupRestoreUtil { @Inject public BackupRestoreUtil(String configIncludeFilter, String configExcludeFilter) { - setFilters(configIncludeFilter, configExcludeFilter); - } - - public BackupRestoreUtil setFilters(String configIncludeFilter, String configExcludeFilter) { includeFilter = getFilter(configIncludeFilter); excludeFilter = getFilter(configExcludeFilter); - logger.info("Exclude filter set: {}", configExcludeFilter); - logger.info("Include filter set: {}", configIncludeFilter); - return this; } public static Optional getLatestValidMetaPath( IMetaProxy metaProxy, DateUtil.DateRange dateRange) { - // Get a list of manifest files. - List metas = metaProxy.findMetaFiles(dateRange); - - // Find a valid manifest file. - for (AbstractBackupPath meta : metas) { - BackupVerificationResult result = metaProxy.isMetaFileValid(meta); - if (result.valid) { - return Optional.of(meta); - } - } - - return Optional.empty(); + return metaProxy + .findMetaFiles(dateRange) + .stream() + .filter(meta -> metaProxy.isMetaFileValid(meta).valid) + .findFirst(); } - public static List getAllFiles( + public static List getMostRecentSnapshotPaths( AbstractBackupPath latestValidMetaFile, - DateUtil.DateRange dateRange, IMetaProxy metaProxy, Provider pathProvider) throws Exception { - // Download the meta.json file. Path metaFile = metaProxy.downloadMetaFile(latestValidMetaFile); - // Parse meta.json file to find the files required to download from this snapshot. - List allFiles = + List snapshotPaths = metaProxy .getSSTFilesFromMeta(metaFile) .stream() @@ -96,50 +75,43 @@ public static List getAllFiles( return path; }) .collect(Collectors.toList()); - FileUtils.deleteQuietly(metaFile.toFile()); + return snapshotPaths; + } - // Download incremental SSTables after the snapshot meta file. + public static List getIncrementalPaths( + AbstractBackupPath latestValidMetaFile, + DateUtil.DateRange dateRange, + IMetaProxy metaProxy) { Instant snapshotTime; if (metaProxy instanceof MetaV2Proxy) snapshotTime = latestValidMetaFile.getLastModified(); else snapshotTime = latestValidMetaFile.getTime().toInstant(); - DateUtil.DateRange incrementalDateRange = new DateUtil.DateRange(snapshotTime, dateRange.getEndTime()); - Iterator incremental = metaProxy.getIncrementals(incrementalDateRange); - while (incremental.hasNext()) allFiles.add(incremental.next()); - - return allFiles; + List incrementalPaths = new ArrayList<>(); + metaProxy.getIncrementals(incrementalDateRange).forEachRemaining(incrementalPaths::add); + return incrementalPaths; } - public static final Map> getFilter(String inputFilter) + public static Map> getFilter(String inputFilter) throws IllegalArgumentException { if (StringUtils.isEmpty(inputFilter)) return null; - - final Map> columnFamilyFilter = - new HashMap<>(); // key: keyspace, value: a list of CFs within the keyspace - + final Map> columnFamilyFilter = new HashMap<>(); String[] filters = inputFilter.split(","); - for (String cfFilter : - filters) { // process filter of form keyspace.* or keyspace.columnfamily + for (String cfFilter : filters) { if (columnFamilyFilterPattern.matcher(cfFilter).find()) { - String[] filter = cfFilter.split("\\."); String keyspaceName = filter[0]; String columnFamilyName = filter[1]; - if (columnFamilyName.contains("-")) columnFamilyName = columnFamilyName.substring(0, columnFamilyName.indexOf("-")); - List existingCfs = columnFamilyFilter.getOrDefault(keyspaceName, new ArrayList<>()); if (!columnFamilyName.equalsIgnoreCase("*")) existingCfs.add(columnFamilyName); columnFamilyFilter.put(keyspaceName, existingCfs); - } else { throw new IllegalArgumentException( - "Column family filter format is not valid. Format needs to be \"keyspace.columnfamily\". Invalid input: " - + cfFilter); + "Invalid format: " + cfFilter + ". \"keyspace.columnfamily\" is required."); } } return columnFamilyFilter; @@ -154,34 +126,19 @@ public static final Map> getFilter(String inputFilter) */ public final boolean isFiltered(String keyspace, String columnFamilyDir) { if (StringUtils.isEmpty(keyspace) || StringUtils.isEmpty(columnFamilyDir)) return false; - String columnFamilyName = columnFamilyDir.split("-")[0]; - // column family is in list of global CF filter if (FILTER_COLUMN_FAMILY.containsKey(keyspace) && FILTER_COLUMN_FAMILY.get(keyspace).contains(columnFamilyName)) return true; - if (excludeFilter != null) if (excludeFilter.containsKey(keyspace) && (excludeFilter.get(keyspace).isEmpty() || excludeFilter.get(keyspace).contains(columnFamilyName))) { - logger.debug( - "Skipping: keyspace: {}, CF: {} is part of exclude list.", - keyspace, - columnFamilyName); return true; } - if (includeFilter != null) - if (!(includeFilter.containsKey(keyspace) + return !(includeFilter.containsKey(keyspace) && (includeFilter.get(keyspace).isEmpty() - || includeFilter.get(keyspace).contains(columnFamilyName)))) { - logger.debug( - "Skipping: keyspace: {}, CF: {} is not part of include list.", - keyspace, - columnFamilyName); - return true; - } - + || includeFilter.get(keyspace).contains(columnFamilyName))); return false; } } diff --git a/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java b/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java index 99c52abb2..2f79852b2 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/IMetaProxy.java @@ -77,10 +77,8 @@ public interface IMetaProxy { * * @param dateRange the time period to scan in the remote file system for incremental files. * @return iterator containing the list of path on the remote file system satisfying criteria. - * @throws BackupRestoreException if there is an issue contacting remote file system. */ - Iterator getIncrementals(DateUtil.DateRange dateRange) - throws BackupRestoreException; + Iterator getIncrementals(DateUtil.DateRange dateRange); /** * Validate that all the files mentioned in the meta file actually exists on remote file system. diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java index 348311918..3ed499336 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV1Proxy.java @@ -169,8 +169,7 @@ public List getSSTFilesFromMeta(Path localMetaPath) throws Exception { } @Override - public Iterator getIncrementals(DateUtil.DateRange dateRange) - throws BackupRestoreException { + public Iterator getIncrementals(DateUtil.DateRange dateRange) { String prefix = fs.getPrefix().toString(); Iterator iterator = fs.list( diff --git a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java index 1a810f476..288e644b1 100644 --- a/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java +++ b/priam/src/main/java/com/netflix/priam/backupv2/MetaV2Proxy.java @@ -79,8 +79,7 @@ private String getMatch( } @Override - public Iterator getIncrementals(DateUtil.DateRange dateRange) - throws BackupRestoreException { + public Iterator getIncrementals(DateUtil.DateRange dateRange) { String incrementalPrefix = getMatch(dateRange, AbstractBackupPath.BackupFileType.SST_V2); String marker = getMatch( diff --git a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java index 5b2576973..fcb03d29d 100644 --- a/priam/src/main/java/com/netflix/priam/config/IConfiguration.java +++ b/priam/src/main/java/com/netflix/priam/config/IConfiguration.java @@ -1178,6 +1178,11 @@ default long getCompressionTransitionEpochMillis() { /** @return whether to enable auto_snapshot */ boolean getAutoSnapshot(); + /** @return whether incremental backups should be skipped in a restore */ + default boolean skipIncrementalRestore() { + return false; + } + /** * Escape hatch for getting any arbitrary property by key This is useful so we don't have to * keep adding methods to this interface for every single configuration option ever. Also diff --git a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java index 7b11807d5..b55a1edef 100644 --- a/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java +++ b/priam/src/main/java/com/netflix/priam/resources/BackupServletV2.java @@ -147,8 +147,11 @@ public Response list(@PathParam("daterange") String daterange) throws Exception return Response.ok("No valid meta found!").build(); } List allFiles = - BackupRestoreUtil.getAllFiles( - latestValidMetaFile.get(), dateRange, metaProxy, pathProvider); + BackupRestoreUtil.getMostRecentSnapshotPaths( + latestValidMetaFile.get(), metaProxy, pathProvider); + allFiles.addAll( + BackupRestoreUtil.getIncrementalPaths( + latestValidMetaFile.get(), dateRange, metaProxy)); return Response.ok( GsonJsonSerializer.getGson() diff --git a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java index 60f69d14b..1074206e5 100644 --- a/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java +++ b/priam/src/main/java/com/netflix/priam/restore/AbstractRestore.java @@ -114,10 +114,6 @@ public static final boolean isRestoreEnabled(IConfiguration conf, InstanceInfo i return (isRestoreMode && isBackedupRac); } - public void setRestoreConfiguration(String restoreIncludeCFList, String restoreExcludeCFList) { - backupRestoreUtil.setFilters(restoreIncludeCFList, restoreExcludeCFList); - } - private List> download( Iterator fsIterator, boolean waitForCompletion) throws Exception { List> futureList = new ArrayList<>(); @@ -247,8 +243,13 @@ public void restore(DateUtil.DateRange dateRange) throws Exception { .setSnapshotMetaFile(latestValidMetaFile.get().getRemotePath()); List allFiles = - BackupRestoreUtil.getAllFiles( - latestValidMetaFile.get(), dateRange, metaProxy, pathProvider); + BackupRestoreUtil.getMostRecentSnapshotPaths( + latestValidMetaFile.get(), metaProxy, pathProvider); + if (!config.skipIncrementalRestore()) { + allFiles.addAll( + BackupRestoreUtil.getIncrementalPaths( + latestValidMetaFile.get(), dateRange, metaProxy)); + } // Download snapshot which is listed in the meta file. List> futureList = new ArrayList<>();