Skip to content

Commit

Permalink
pnfsmanager: refactor Chimera(Osm|Enstore)StorageInfoExtractor
Browse files Browse the repository at this point in the history 
Motivation:

Patch baece52 started refactoring ChimeraOsmStorageInfoExtractor to
make the code easier to understand and maintain.  The patch continues
that work focusing more on ChimeraEnstoreStorageInfoExtractor.

Modification:

Refactor code to avoid unnecessary complexity.  There should be no
change in the result of the code.

Result:

No user or admin observable changes.

Target: master
Requires-book: no
Requires-notes: no
Patch: https://rb.dcache.org/r/12600/
Acked-by: Albert Rossi
  • Loading branch information
@paulmillar
paulmillar committed Oct 5, 2020
1 parent 947216b commit f3a7b7a
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 109 deletions.
133 changes: 61 additions & 72 deletions ...himera/src/main/java/org/dcache/chimera/namespace/ChimeraEnstoreStorageInfoExtractor.java
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
package org.dcache.chimera.namespace;

import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import org.dcache.chimera.FileState;
import org.springframework.web.util.UriComponentsBuilder;
import org.springframework.web.util.UriUtils;

import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

import diskCacheV111.util.AccessLatency;
Expand All @@ -22,98 +19,90 @@

import org.dcache.chimera.ChimeraFsException;
import org.dcache.chimera.StorageGenericLocation;
import org.dcache.chimera.posix.Stat;


public class ChimeraEnstoreStorageInfoExtractor extends ChimeraHsmStorageInfoExtractor {

public class ChimeraEnstoreStorageInfoExtractor
extends ChimeraHsmStorageInfoExtractor
{
public ChimeraEnstoreStorageInfoExtractor(AccessLatency defaultAL,
RetentionPolicy defaultRP) {
super(defaultAL,defaultRP);
}

@Override
public StorageInfo getFileStorageInfo(ExtendedInode inode) throws CacheException {
EnstoreStorageInfo info;

public StorageInfo getFileStorageInfo(ExtendedInode inode)
throws CacheException {
try {
Stat stat = inode.stat();
boolean isNew = stat.getState() == FileState.CREATED;

info = (EnstoreStorageInfo) getDirStorageInfo(inode);
if (!isNew) {
List<String> locations = inode.getLocations(StorageGenericLocation.TAPE);
if (!locations.isEmpty()) {
info = new EnstoreStorageInfo(info.getStorageGroup(), info.getFileFamily());
for (String location : locations) {
UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(location);
URI uri = builder.build(isEncoded(location)).toUri();
info.addLocation(uri);
String queryString = uri.getQuery();
if (!Strings.isNullOrEmpty(queryString)) {
for (String part : uri.getQuery().split("&")) {
String[] data = part.split("=");
String key = data[0];
String value = (data.length == 2 ? data[1] : "");
switch (key) {
case "bfid":
info.setBitfileId(value);
break;
case "volume":
info.setVolume(value);
break;
case "location_cookie":
info.setLocation(value);
break;
case "original_name":
info.setPath(value);
break;
}
}
EnstoreStorageInfo info = getDirStorageInfo(inode);

if (inode.stat().getState() == FileState.CREATED) {
return info;
}

List<String> tapeLocations = inode.getLocations(StorageGenericLocation.TAPE);
if (!tapeLocations.isEmpty()) {
info.clearKeys();

for (String location : tapeLocations) {
URI uri = UriComponentsBuilder.fromUriString(location)
.build(isEncoded(location)).toUri();
info.addLocation(uri);

String queryString = Strings.nullToEmpty(uri.getQuery());
for (String part : queryString.split("&")) {
String[] data = part.split("="); // REVISIT what if 'part' contains multiple '='?
String value = data.length == 2 ? data[1] : "";
switch (data[0]) {
case "bfid":
info.setBitfileId(value);
break;
case "volume":
info.setVolume(value);
break;
case "location_cookie":
info.setLocation(value);
break;
case "original_name":
info.setPath(value);
break;
}
}
}
}
info.setIsNew(isNew);
}
catch (ChimeraFsException e) {

info.setIsNew(false);
return info;
} catch (ChimeraFsException e) {
throw new CacheException(e.getMessage());
}
return info;
}

@Override
public StorageInfo getDirStorageInfo(ExtendedInode inode) throws CacheException {
ExtendedInode dirInode;
if (!inode.isDirectory()) {
dirInode = inode.getParent();
if (dirInode == null) {
throw new FileNotFoundCacheException("file unlinked");
}
}
else {
dirInode = inode;
}
Map<String, String> hash = new HashMap<>();
ImmutableList<String> OSMTemplate = dirInode.getTag("OSMTemplate");
ImmutableList<String> group = dirInode.getTag("storage_group");
ImmutableList<String> family = dirInode.getTag("file_family");

for (String line: OSMTemplate) {
StringTokenizer st = new StringTokenizer(line);
if (st.countTokens() >= 2) {
hash.put(st.nextToken().intern(), st.nextToken());
}
public EnstoreStorageInfo getDirStorageInfo(ExtendedInode inode)
throws CacheException {
ExtendedInode directory = inode.isDirectory() ? inode: inode.getParent();

if (directory == null) {
throw new FileNotFoundCacheException("file unlinked");
}
String sg = getFirstLine(group).map(String::intern).orElse("none");
String ff = getFirstLine(family).map(String::intern).orElse("none");
EnstoreStorageInfo info = new EnstoreStorageInfo(sg,ff);
info.addKeys(hash);

List<String> groupTag = directory.getTag("storage_group");
String storageGroup = getFirstLine(groupTag).map(String::intern).orElse("none");

List<String> familyTag = directory.getTag("file_family");
String fileFamily = getFirstLine(familyTag).map(String::intern).orElse("none");

EnstoreStorageInfo info = new EnstoreStorageInfo(storageGroup, fileFamily);

directory.getTag("OSMTemplate").stream()
.map(StringTokenizer::new)
.filter(t -> t.countTokens() >= 2)
.forEach(t -> info.setKey(t.nextToken().intern(), t.nextToken()));

return info;
}

private static boolean isEncoded(String s) {
return !s.equals(UriUtils.decode(s, StandardCharsets.UTF_8));
}

}
2 changes: 1 addition & 1 deletion ...he-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraHsmStorageInfoExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ public void setStorageInfo(FsInode inode, StorageInfo dCacheStorageInfo) throws
}
}

protected static Optional<String> getFirstLine(ImmutableList<String> lines)
protected static Optional<String> getFirstLine(List<String> lines)
{
if (!lines.isEmpty()) {
String line = lines.get(0).trim();
Expand Down
61 changes: 25 additions & 36 deletions ...he-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraOsmStorageInfoExtractor.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package org.dcache.chimera.namespace;

import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

import diskCacheV111.util.AccessLatency;
Expand All @@ -20,7 +20,6 @@
import org.dcache.chimera.ChimeraFsException;
import org.dcache.chimera.FileState;
import org.dcache.chimera.StorageGenericLocation;
import org.dcache.chimera.posix.Stat;
import org.dcache.chimera.store.InodeStorageInformation;


Expand All @@ -37,12 +36,8 @@ public ChimeraOsmStorageInfoExtractor(AccessLatency defaultAL,
public StorageInfo getFileStorageInfo(ExtendedInode inode)
throws CacheException {
try {
Stat stat = inode.statCache();

if (stat.getState() == FileState.CREATED) {
StorageInfo info = getDirStorageInfo(inode);
info.setIsNew(true);
return info;
if (inode.statCache().getState() == FileState.CREATED) {
return getDirStorageInfo(inode);
}

List<String> tapeLocations = inode.getLocations(StorageGenericLocation.TAPE);
Expand Down Expand Up @@ -74,39 +69,33 @@ public StorageInfo getFileStorageInfo(ExtendedInode inode)
}

@Override
public StorageInfo getDirStorageInfo(ExtendedInode inode) throws CacheException {
ExtendedInode dirInode;
if (!inode.isDirectory()) {
dirInode = inode.getParent();
if (dirInode == null) {
throw new FileNotFoundCacheException("file unlinked");
}
}
else {
dirInode = inode;
public StorageInfo getDirStorageInfo(ExtendedInode inode)
throws CacheException {
ExtendedInode directory = inode.isDirectory() ? inode: inode.getParent();

if (directory == null) {
throw new FileNotFoundCacheException("file unlinked");
}
HashMap<String, String> hash = new HashMap<>();
String store = null;
ImmutableList<String> OSMTemplate = dirInode.getTag("OSMTemplate");
if (!OSMTemplate.isEmpty()) {
for (String line: OSMTemplate) {
StringTokenizer st = new StringTokenizer(line);
if (st.countTokens() < 2) {
continue;
}
hash.put(st.nextToken().intern(), st.nextToken());
}
store = hash.get("StoreName");
if (store == null) {
throw new CacheException(37, "StoreName not found in template");
}

List<String> osmTemplateTag = directory.getTag("OSMTemplate");

Map<String, String> hash = new HashMap<>();
osmTemplateTag.stream()
.map(StringTokenizer::new)
.filter(t -> t.countTokens() >= 2)
.forEach(t -> hash.setKey(t.nextToken().intern(), t.nextToken()));

String store = hash.get("StoreName");

if (store == null && !osmTemplateTag.isEmpty()) {
throw new CacheException(37, "StoreName not found in template");
}

ImmutableList<String> sGroup = dirInode.getTag("sGroup");
String group = getFirstLine(sGroup).map(String::intern).orElse(null);
List<String> sGroupTag = directory.getTag("sGroup");
String group = getFirstLine(sGroupTag).map(String::intern).orElse(null);

OSMStorageInfo info = new OSMStorageInfo(store, group);
info.addKeys(hash);
return info;
}

}
4 changes: 4 additions & 0 deletions modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/GenericStorageInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ public void addKeys(Map<String, String> keys) {
_keyHash.putAll(keys);
}

public void clearKeys() {
_keyHash.clear();
}

@Override
public void addLocation(URI newLocation) {
_locations.add(newLocation);
Expand Down

0 comments on commit f3a7b7a

Please sign in to comment.