Skip to content
Permalink
Browse files
List snapshots of dropped tables
Patch by Paulo Motta; Reviewed by Stefan Miklosovic and Brandon Williams or CASSANDRA-16843
  • Loading branch information
pauloricardomg committed Apr 29, 2022
1 parent 458bfd1 commit 31aa17a2a3b18bdda723123cad811f075287807d
Showing 17 changed files with 707 additions and 120 deletions.
@@ -1,4 +1,5 @@
4.1
* List snapshots of dropped tables (CASSANDRA-16843)
* Add information whether sstables are dropped to SchemaChangeListener (CASSANDRA-17582)
* Add a pluggable memtable API (CEP-11 / CASSANDRA-17034)
* Save sstable id as string in activity table (CASSANDRA-17585)
@@ -2060,8 +2060,8 @@ protected TableSnapshot createSnapshot(String tag, boolean ephemeral, DurationSp
snapshotDirs.add(ephemeralSnapshotMarker.parent().toAbsolute()); // marker may create empty snapshot dir
}

TableSnapshot snapshot = new TableSnapshot(metadata.keyspace, metadata.name, tag, manifest.createdAt,
manifest.expiresAt, snapshotDirs, directories::getTrueAllocatedSizeIn);
TableSnapshot snapshot = new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(), tag,
manifest.createdAt, manifest.expiresAt, snapshotDirs);

StorageService.instance.addSnapshot(snapshot);
return snapshot;
@@ -556,14 +556,19 @@ public File getSnapshotManifestFile(String snapshotName)
return getSnapshotManifestFile(snapshotDir);
}

protected static File getSnapshotManifestFile(File snapshotDir)
public static File getSnapshotManifestFile(File snapshotDir)
{
return new File(snapshotDir, "manifest.json");
}

public File getSnapshotSchemaFile(String snapshotName)
{
File snapshotDir = getSnapshotDirectory(getDirectoryForNewSSTables(), snapshotName);
return getSnapshotSchemaFile(snapshotDir);
}

public static File getSnapshotSchemaFile(File snapshotDir)
{
return new File(snapshotDir, "schema.cql");
}

@@ -981,8 +986,8 @@ public Map<String, TableSnapshot> listSnapshots()
protected TableSnapshot buildSnapshot(String tag, SnapshotManifest manifest, Set<File> snapshotDirs) {
Instant createdAt = manifest == null ? null : manifest.createdAt;
Instant expiresAt = manifest == null ? null : manifest.expiresAt;
return new TableSnapshot(metadata.keyspace, metadata.name, tag, createdAt, expiresAt, snapshotDirs,
this::getTrueAllocatedSizeIn);
return new TableSnapshot(metadata.keyspace, metadata.name, metadata.id.asUUID(), tag, createdAt, expiresAt,
snapshotDirs);
}

@VisibleForTesting
@@ -1155,7 +1160,7 @@ public long getTrueAllocatedSizeIn(File snapshotDir)
if (!snapshotDir.isDirectory())
return 0;

SSTableSizeSummer visitor = new SSTableSizeSummer(snapshotDir, sstableLister(OnTxnErr.THROW).listFiles());
SSTableSizeSummer visitor = new SSTableSizeSummer(sstableLister(OnTxnErr.THROW).listFiles());
try
{
Files.walkFileTree(snapshotDir.toPath(), visitor);
@@ -1244,10 +1249,9 @@ private static String join(String... s)
private class SSTableSizeSummer extends DirectorySizeCalculator
{
private final Set<String> toSkip;
SSTableSizeSummer(File path, List<File> files)
SSTableSizeSummer(List<File> files)
{
super(path);
toSkip = files.stream().map(f -> f.name()).collect(Collectors.toSet());
toSkip = files.stream().map(File::name).collect(Collectors.toSet());
}

@Override
@@ -78,7 +78,7 @@ public static void from(TableSnapshot details, TabularDataSupport result)
String createdAt = safeToString(details.getCreatedAt());
String expiresAt = safeToString(details.getExpiresAt());
result.put(new CompositeDataSupport(COMPOSITE_TYPE, ITEM_NAMES,
new Object[]{ details.getTag(), details.getKeyspace(), details.getTable(), liveSize, totalSize, createdAt, expiresAt }));
new Object[]{ details.getTag(), details.getKeyspaceName(), details.getTableName(), liveSize, totalSize, createdAt, expiresAt }));
}
catch (OpenDataException e)
{
@@ -274,9 +274,11 @@ private static class CDCSizeTracker extends DirectorySizeCalculator
// track the total size between two dictionary size calculations
private final AtomicLong sizeInProgress;

private final File path;

CDCSizeTracker(CommitLogSegmentManagerCDC segmentManager, File path)
{
super(path);
this.path = path;
this.segmentManager = segmentManager;
this.sizeInProgress = new AtomicLong(0);
}
@@ -73,6 +73,7 @@
import org.apache.cassandra.locator.ReplicaCollection.Builder.Conflict;
import org.apache.cassandra.schema.Keyspaces;
import org.apache.cassandra.service.disk.usage.DiskUsageBroadcaster;
import org.apache.cassandra.service.snapshot.SnapshotLoader;
import org.apache.cassandra.utils.concurrent.Future;
import org.apache.cassandra.schema.TableId;
import org.apache.cassandra.utils.concurrent.FutureCombiner;
@@ -4135,24 +4136,26 @@ public void clearSnapshot(String tag, String... keyspaceNames) throws IOExceptio

public Map<String, TabularData> getSnapshotDetails(Map<String, String> options)
{
boolean skipExpiring = options != null && Boolean.parseBoolean(options.getOrDefault("no_ttl", "false"));

SnapshotLoader loader = new SnapshotLoader();
Map<String, TabularData> snapshotMap = new HashMap<>();
for (Keyspace keyspace : Keyspace.all())

for (TableSnapshot snapshot : loader.loadSnapshots())
{
for (ColumnFamilyStore cfStore : keyspace.getColumnFamilyStores())
{
for (Map.Entry<String, TableSnapshot> snapshotDetail : TableSnapshot.filter(cfStore.listSnapshots(), options).entrySet())
{
TabularDataSupport data = (TabularDataSupport) snapshotMap.get(snapshotDetail.getKey());
if (data == null)
{
data = new TabularDataSupport(SnapshotDetailsTabularData.TABULAR_TYPE);
snapshotMap.put(snapshotDetail.getKey(), data);
}
if (skipExpiring && snapshot.isExpiring())
continue;

SnapshotDetailsTabularData.from(snapshotDetail.getValue(), data);
}
TabularDataSupport data = (TabularDataSupport) snapshotMap.get(snapshot.getTag());
if (data == null)
{
data = new TabularDataSupport(SnapshotDetailsTabularData.TABULAR_TYPE);
snapshotMap.put(snapshot.getTag(), data);
}

SnapshotDetailsTabularData.from(snapshot, data);
}

return snapshotMap;
}

@@ -0,0 +1,137 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.cassandra.service.snapshot;

import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.util.File;

import static org.apache.cassandra.db.Directories.SNAPSHOT_SUBDIR;
import static org.apache.cassandra.service.snapshot.TableSnapshot.buildSnapshotId;

/**
* Loads snapshot metadata from data directories
*/
public class SnapshotLoader extends SimpleFileVisitor<Path>
{
private static final Logger logger = LoggerFactory.getLogger(SnapshotLoader.class);

static final Pattern SNAPSHOT_DIR_PATTERN = Pattern.compile("(?<keyspace>\\w+)/(?<tableName>\\w+)\\-(?<tableId>[0-9a-f]{32})/snapshots/(?<tag>[\\w-]+)$");

private final Collection<Path> dataDirectories;
private final Map<String, TableSnapshot.Builder> snapshots = new HashMap<>();

public SnapshotLoader()
{
this(DatabaseDescriptor.getAllDataFileLocations());
}

public SnapshotLoader(String[] dataDirectories)
{
this.dataDirectories = Arrays.stream(dataDirectories).map(Paths::get).collect(Collectors.toList());
}

public SnapshotLoader(Collection<Path> dataDirs)
{
this.dataDirectories = dataDirs;
}

public Set<TableSnapshot> loadSnapshots()
{
for (Path dataDir : dataDirectories)
{
try
{
Files.walkFileTree(dataDir, Collections.EMPTY_SET, 5, this);
}
catch (IOException e)
{
throw new RuntimeException(String.format("Error while loading snapshots from %s", dataDir));
}
}
return snapshots.values().stream().map(TableSnapshot.Builder::build).collect(Collectors.toSet());
}

public FileVisitResult preVisitDirectory(Path subdir, BasicFileAttributes attrs)
{
if (subdir.getParent().getFileName().toString().equals(SNAPSHOT_SUBDIR))
{
logger.trace("Processing directory " + subdir);
Matcher snapshotDirMatcher = SNAPSHOT_DIR_PATTERN.matcher(subdir.toString());
if (snapshotDirMatcher.find())
{
try
{
loadSnapshotFromDir(snapshotDirMatcher, subdir);
} catch (Throwable e)
{
logger.warn("Could not load snapshot from {}.", subdir, e);
}
}
return FileVisitResult.SKIP_SUBTREE;
}

return subdir.getFileName().equals(Directories.BACKUPS_SUBDIR)
? FileVisitResult.SKIP_SUBTREE
: FileVisitResult.CONTINUE;
}

private void loadSnapshotFromDir(Matcher snapshotDirMatcher, Path snapshotDir)
{
String keyspaceName = snapshotDirMatcher.group("keyspace");
String tableName = snapshotDirMatcher.group("tableName");
UUID tableId = parseUUID(snapshotDirMatcher.group("tableId"));
String tag = snapshotDirMatcher.group("tag");
String snapshotId = buildSnapshotId(keyspaceName, tableName, tableId, tag);
TableSnapshot.Builder builder = snapshots.computeIfAbsent(snapshotId, k -> new TableSnapshot.Builder(keyspaceName, tableName, tableId, tag));
builder.addSnapshotDir(new File(snapshotDir));
}

/**
* Given an UUID string without dashes (ie. c7e513243f0711ec9bbc0242ac130002)
* return an UUID object (ie. c7e51324-3f07-11ec-9bbc-0242ac130002)
*/
protected static UUID parseUUID(String uuidWithoutDashes) throws IllegalArgumentException
{
assert uuidWithoutDashes.length() == 32 && !uuidWithoutDashes.contains("-");
String dashedUUID = uuidWithoutDashes.replaceFirst("([0-9a-f]{8})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]{4})([0-9a-f]+)", "$1-$2-$3-$4-$5");
return UUID.fromString(dashedUUID);
}
}

0 comments on commit 31aa17a

Please sign in to comment.