Skip to content
Permalink
Browse files
Create TabletDirectory classes (#2688)
* Create TabletDirectory and RelativeTabletDirectory
* Move Reference class and improve it so other classes can be children
* Add validate method to StoredTabletFile
* Use TabletDirectory in TabletFile
* Refactor some GC code
  • Loading branch information
milleruntime committed May 18, 2022
1 parent cdc957f commit 6c5e5683eb8e267aba60e9e93af3ea642cfb9ddb
Showing 9 changed files with 172 additions and 47 deletions.
@@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.core.metadata;

import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.metadata.schema.MetadataSchema;

/**
* A reference to a tablet file or directory.
*/
public class Reference {
// parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003"
public final TableId tableId; // 2a
public final String tabletDir; // t-0003

// the exact string that is stored in the metadata
public final String metadataEntry;

public Reference(TableId tableId, String metadataEntry) {
MetadataSchema.TabletsSection.ServerColumnFamily.validateDirCol(tableId.canonical());
this.tableId = tableId;
this.metadataEntry = metadataEntry;
this.tabletDir = metadataEntry;
}
}
@@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.core.metadata;

import org.apache.accumulo.core.data.TableId;

/**
* Part of the Tablet File path that is definitely a directory.
*/
public class RelativeTabletDirectory extends Reference {

public RelativeTabletDirectory(TableId tableId, String dirName) {
super(tableId, dirName);
}
}
@@ -59,4 +59,17 @@ public String getMetaUpdateDelete() {
public Text getMetaUpdateDeleteText() {
return new Text(getMetaUpdateDelete());
}

/**
* Validate that the provided reference matches what is in the metadata table.
*
* @param reference
* the relative path to check against
*/
public void validate(String reference) {
if (!metadataEntry.equals(reference)) {
throw new IllegalStateException("The reference " + reference
+ " does not match what was in the metadata: " + metadataEntry);
}
}
}
@@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.accumulo.core.metadata;

import org.apache.accumulo.core.data.TableId;

/**
* The Tablet directory that may exist in the metadata table.
*/
public class TabletDirectory extends RelativeTabletDirectory {
// parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003"
private final String volume; // hdfs://1.2.3.4/accumulo

public TabletDirectory(String volume, TableId tableId, String tabletDir) {
super(tableId, tabletDir);
this.volume = volume;
}

public String getVolume() {
return volume;
}

public TableId getTableId() {
return tableId;
}

public String getTabletDir() {
return tabletDir;
}

@Override
public String toString() {
return tabletDir;
}
}
@@ -18,9 +18,10 @@
*/
package org.apache.accumulo.core.metadata;

import static org.apache.accumulo.core.Constants.HDFS_TABLES_DIR;

import java.util.Objects;

import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
import org.apache.hadoop.fs.Path;
@@ -40,9 +41,7 @@
*/
public class TabletFile implements Comparable<TabletFile> {
// parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003/C0004.rf"
private final String volume; // hdfs://1.2.3.4/accumulo
private final TableId tableId; // 2a
private final String tabletDir; // t-0003
private final TabletDirectory tabletDir; // hdfs://1.2.3.4/accumulo/tables/2a/t-0003
private final String fileName; // C0004.rf
protected final Path metaPath;
private final String normalizedPath;
@@ -60,35 +59,32 @@ public TabletFile(Path metaPath) {
ServerColumnFamily.validateDirCol(fileName);

Path tabletDirPath = Objects.requireNonNull(metaPath.getParent(), errorMsg);
this.tabletDir = tabletDirPath.getName();
ServerColumnFamily.validateDirCol(tabletDir);

Path tableIdPath = Objects.requireNonNull(tabletDirPath.getParent(), errorMsg);
this.tableId = TableId.of(tableIdPath.getName());
ServerColumnFamily.validateDirCol(tableId.canonical());
var id = tableIdPath.getName();

Path tablePath = Objects.requireNonNull(tableIdPath.getParent(), errorMsg);
String tpString = "/" + tablePath.getName();
Preconditions.checkArgument(tpString.equals(Constants.HDFS_TABLES_DIR), errorMsg);
Preconditions.checkArgument(tpString.equals(HDFS_TABLES_DIR), errorMsg);

Path volumePath = Objects.requireNonNull(tablePath.getParent(), errorMsg);
Preconditions.checkArgument(volumePath.toUri().getScheme() != null, errorMsg);
this.volume = volumePath.toString();
var volume = volumePath.toString();

this.normalizedPath = volume + Constants.HDFS_TABLES_DIR + "/" + tableId.canonical() + "/"
+ tabletDir + "/" + fileName;
this.tabletDir = new TabletDirectory(volume, TableId.of(id), tabletDirPath.getName());
this.normalizedPath = volume + HDFS_TABLES_DIR + "/" + id + "/" + tabletDir + "/" + fileName;
}

public String getVolume() {
return volume;
return tabletDir.getVolume();
}

public TableId getTableId() {
return tableId;
return tabletDir.getTableId();
}

public String getTabletDir() {
return tabletDir;
return tabletDir.getTabletDir();
}

public String getFileName() {
@@ -46,6 +46,8 @@
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.metadata.Reference;
import org.apache.accumulo.core.metadata.RelativeTabletDirectory;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.TabletFileUtil;
import org.apache.accumulo.core.metadata.schema.Ample;
@@ -146,10 +148,10 @@ public Stream<Reference> getReferences() {

return tabletStream.flatMap(tm -> {
Stream<Reference> refs = Stream.concat(tm.getFiles().stream(), tm.getScans().stream())
.map(f -> new Reference(tm.getTableId(), f.getMetaUpdateDelete(), false));
.map(f -> new Reference(tm.getTableId(), f.getMetaUpdateDelete()));
if (tm.getDirName() != null) {
refs =
Stream.concat(refs, Stream.of(new Reference(tm.getTableId(), tm.getDirName(), true)));
refs = Stream.concat(refs,
Stream.of(new RelativeTabletDirectory(tm.getTableId(), tm.getDirName())));
}
return refs;
});
@@ -34,9 +34,10 @@
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.metadata.Reference;
import org.apache.accumulo.core.metadata.RelativeTabletDirectory;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.gc.GarbageCollectionEnvironment.Reference;
import org.apache.accumulo.server.replication.proto.Replication.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,6 +107,7 @@ private String makeRelative(String path, int expectedLen) {
throw new IllegalArgumentException(path);
}

log.trace("{} -> {} expectedLen = {}", path, relPath, expectedLen);
return relPath;
}

@@ -132,36 +134,35 @@ private void removeCandidatesInUse(GarbageCollectionEnvironment gce,
while (iter.hasNext()) {
Reference ref = iter.next();

if (ref.isDir) {
String tableID = ref.id.toString();
String dirName = ref.ref;
ServerColumnFamily.validateDirCol(dirName);
if (ref instanceof RelativeTabletDirectory) {
var dirReference = (RelativeTabletDirectory) ref;
ServerColumnFamily.validateDirCol(dirReference.tabletDir);

String dir = "/" + tableID + "/" + dirName;
String dir = "/" + dirReference.tableId + "/" + dirReference.tabletDir;

dir = makeRelative(dir, 2);

if (candidateMap.remove(dir) != null)
log.debug("Candidate was still in use: {}", dir);
} else {

String reference = ref.ref;
String reference = ref.metadataEntry;
if (reference.startsWith("/")) {
reference = "/" + ref.id + reference;
log.debug("Candidate {} has a relative path, prepend tableId {}", reference, ref.tableId);
reference = "/" + ref.tableId + ref.metadataEntry;
} else if (!reference.contains(":") && !reference.startsWith("../")) {
throw new RuntimeException("Bad file reference " + reference);
}

reference = makeRelative(reference, 3);
String relativePath = makeRelative(reference, 3);

// WARNING: This line is EXTREMELY IMPORTANT.
// You MUST REMOVE candidates that are still in use
if (candidateMap.remove(reference) != null)
log.debug("Candidate was still in use: {}", reference);
if (candidateMap.remove(relativePath) != null)
log.debug("Candidate was still in use: {}", relativePath);

String dir = reference.substring(0, reference.lastIndexOf('/'));
String dir = relativePath.substring(0, relativePath.lastIndexOf('/'));
if (candidateMap.remove(dir) != null)
log.debug("Candidate was still in use: {}", reference);
log.debug("Candidate was still in use: {}", relativePath);
}
}
}
@@ -32,6 +32,7 @@
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.Reference;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
@@ -65,18 +66,6 @@ public interface GarbageCollectionEnvironment {
*/
Stream<String> getBlipPaths() throws TableNotFoundException;

static class Reference {
public final TableId id;
public final String ref;
public final boolean isDir;

Reference(TableId id, String ref, boolean isDir) {
this.id = id;
this.ref = ref;
this.isDir = isDir;
}
}

/**
* Fetches the references to files, {@link DataFileColumnFamily#NAME} or
* {@link ScanFileColumnFamily#NAME}, from tablets
@@ -37,6 +37,8 @@

import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.metadata.Reference;
import org.apache.accumulo.core.metadata.RelativeTabletDirectory;
import org.apache.accumulo.server.replication.proto.Replication.Status;
import org.junit.jupiter.api.Test;

@@ -93,16 +95,15 @@ public void deleteTableDirIfEmpty(TableId tableID) {
}

public void addFileReference(String tableId, String endRow, String file) {
references.put(tableId + ":" + endRow + ":" + file,
new Reference(TableId.of(tableId), file, false));
references.put(tableId + ":" + endRow + ":" + file, new Reference(TableId.of(tableId), file));
}

public void removeFileReference(String tableId, String endRow, String file) {
references.remove(tableId + ":" + endRow + ":" + file);
}

public void addDirReference(String tableId, String endRow, String dir) {
references.put(tableId + ":" + endRow, new Reference(TableId.of(tableId), dir, true));
references.put(tableId + ":" + endRow, new RelativeTabletDirectory(TableId.of(tableId), dir));
}

public void removeDirReference(String tableId, String endRow) {

0 comments on commit 6c5e568

Please sign in to comment.