Skip to content

Commit

Permalink
IGNITE-3405: IGFS: Restricted path modes interleaving, so that now on…
Browse files Browse the repository at this point in the history 
…ly DUAL -> PRIMARY and DUAL -> PROXY paths are possible.
  • Loading branch information
vozerov-gridgain committed Jul 20, 2016
1 parent 51add50 commit 9b55658
Show file tree
Hide file tree
Showing 11 changed files with 276 additions and 577 deletions.
77 changes: 20 additions & 57 deletions modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
View file
Expand Up @@ -94,7 +94,6 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
Expand Down Expand Up @@ -210,7 +209,7 @@ public final class IgfsImpl implements IgfsEx {
Map<String, IgfsMode> cfgModes = new LinkedHashMap<>();
Map<String, IgfsMode> dfltModes = new LinkedHashMap<>(4, 1.0f);

if (cfg.isInitializeDefaultPathModes()) {
if (cfg.isInitializeDefaultPathModes() && IgfsUtils.isDualMode(dfltMode)) {
dfltModes.put("/ignite/primary", PRIMARY);

if (secondaryFs != null) {
Expand All @@ -222,8 +221,8 @@ public final class IgfsImpl implements IgfsEx {

cfgModes.putAll(dfltModes);

if (igfsCtx.configuration().getPathModes() != null) {
for (Map.Entry<String, IgfsMode> e : igfsCtx.configuration().getPathModes().entrySet()) {
if (cfg.getPathModes() != null) {
for (Map.Entry<String, IgfsMode> e : cfg.getPathModes().entrySet()) {
if (!dfltModes.containsKey(e.getKey()))
cfgModes.put(e.getKey(), e.getValue());
else
Expand All @@ -249,7 +248,7 @@ public final class IgfsImpl implements IgfsEx {
}
}

modeRslvr = new IgfsModeResolver(dfltMode, modes);
modeRslvr = new IgfsModeResolver(dfltMode, IgfsUtils.preparePathModes(dfltMode, modes));

Object secondaryFsPayload = null;

Expand Down Expand Up @@ -620,7 +619,7 @@ else if (val)
IgfsMode mode = resolveMode(path);

if (mode != PRIMARY) {
assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
assert IgfsUtils.isDualMode(mode);

await(path);

Expand Down Expand Up @@ -671,8 +670,6 @@ else if (val)

IgfsMode mode = resolveMode(src);

Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(src);

if (src.equals(dest))
return null; // Rename to itself is a no-op.

Expand All @@ -689,8 +686,8 @@ else if (val)
throw new IgfsInvalidPathException("Cannot move file to a path with different eviction " +
"exclude setting (need to copy and remove)");

if (!childrenModes.equals(Collections.singleton(PRIMARY))) {
assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
if (mode != PRIMARY) {
assert IgfsUtils.isDualMode(mode); // PROXY mode explicit usage is forbidden.

await(src, dest);

Expand Down Expand Up @@ -721,9 +718,9 @@ else if (val)
if (IgfsPath.SLASH.equals(path.toString()))
return false;

Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(path);
IgfsMode mode = resolveMode(path);

boolean dual = childrenModes.contains(DUAL_SYNC) ||childrenModes.contains(DUAL_ASYNC);
boolean dual = IgfsUtils.isDualMode(mode);;

if (dual)
await(path);
Expand Down Expand Up @@ -767,7 +764,7 @@ else if (val)
if (mode == PRIMARY)
meta.mkdirs(path, props0);
else {
assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
assert IgfsUtils.isDualMode(mode);;

await(path);

Expand All @@ -794,11 +791,9 @@ else if (val)

IgfsMode mode = resolveMode(path);

Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(path);

Collection<String> files = new HashSet<>();

if (childrenModes.contains(DUAL_SYNC) || childrenModes.contains(DUAL_ASYNC)) {
if (IgfsUtils.isDualMode(mode)) {
assert secondaryFs != null;

try {
Expand All @@ -818,11 +813,8 @@ else if (val)

if (fileId != null)
files.addAll(meta.directoryListing(fileId).keySet());
else if (mode == PRIMARY) {
checkConflictWithPrimary(path);

else if (mode == PRIMARY)
throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
}

return F.viewReadOnly(files, new C1<String, IgfsPath>() {
@Override public IgfsPath apply(String e) {
Expand All @@ -847,11 +839,9 @@ else if (mode == PRIMARY) {

IgfsMode mode = resolveMode(path);

Set<IgfsMode> childrenModes = modeRslvr.resolveChildrenModes(path);

Collection<IgfsFile> files = new HashSet<>();

if (childrenModes.contains(DUAL_SYNC) || childrenModes.contains(DUAL_ASYNC)) {
if (IgfsUtils.isDualMode(mode)) {
assert secondaryFs != null;

try {
Expand Down Expand Up @@ -894,11 +884,8 @@ else if (mode == PRIMARY) {
}
}
}
else if (mode == PRIMARY) {
checkConflictWithPrimary(path);

else if (mode == PRIMARY)
throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
}

return files;
}
Expand Down Expand Up @@ -937,7 +924,7 @@ else if (mode == PRIMARY) {
IgfsMode mode = resolveMode(path);

if (mode != PRIMARY) {
assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
assert IgfsUtils.isDualMode(mode);

IgfsSecondaryInputStreamDescriptor desc = meta.openDual(secondaryFs, path, bufSize0);

Expand All @@ -951,11 +938,8 @@ else if (mode == PRIMARY) {

IgfsEntryInfo info = meta.infoForPath(path);

if (info == null) {
checkConflictWithPrimary(path);

if (info == null)
throw new IgfsPathNotFoundException("File not found: " + path);
}

if (!info.isFile())
throw new IgfsPathIsDirectoryException("Failed to open file (not a file): " + path);
Expand Down Expand Up @@ -1089,7 +1073,7 @@ private IgfsOutputStream create0(
IgfsFileWorkerBatch batch;

if (mode != PRIMARY) {
assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
assert IgfsUtils.isDualMode(mode);

await(path);

Expand All @@ -1105,11 +1089,8 @@ private IgfsOutputStream create0(
final IgniteUuid id = ids.get(ids.size() - 1);

if (id == null) {
if (!create) {
checkConflictWithPrimary(path);

if (!create)
throw new IgfsPathNotFoundException("File not found: " + path);
}
}

// Prevent attempt to append to ROOT in early stage:
Expand Down Expand Up @@ -1160,11 +1141,8 @@ private IgfsOutputStream create0(
@Override public Void call() throws Exception {
FileDescriptor desc = getFileDescriptor(path);

if (desc == null) {
checkConflictWithPrimary(path);

if (desc == null)
throw new IgfsPathNotFoundException("Failed to update times (path not found): " + path);
}

// Cannot update times for root.
if (desc.parentId == null)
Expand All @@ -1177,21 +1155,6 @@ private IgfsOutputStream create0(
});
}

/**
* Checks if given path exists in secondary file system and throws exception if so.
*
* @param path Path to check.
* @throws IgniteCheckedException If path exists.
*/
private void checkConflictWithPrimary(IgfsPath path) throws IgniteCheckedException {
if (secondaryFs != null) {
if (secondaryFs.info(path) != null) {
throw new IgfsInvalidPathException("Path mapped to a PRIMARY mode found in secondary file " +
"system. Remove path from secondary file system or change path mapping: " + path);
}
}
}

/** {@inheritDoc} */
@Override public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) {
return affinity(path, start, len, 0L);
Expand All @@ -1218,7 +1181,7 @@ private void checkConflictWithPrimary(IgfsPath path) throws IgniteCheckedExcepti
IgfsEntryInfo info = meta.infoForPath(path);

if (info == null && mode != PRIMARY) {
assert mode == DUAL_SYNC || mode == DUAL_ASYNC;
assert IgfsUtils.isDualMode(mode);
assert secondaryFs != null;

// Synchronize
Expand Down
94 changes: 7 additions & 87 deletions modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
View file
Expand Up @@ -17,13 +17,9 @@

package org.apache.ignite.internal.processors.igfs;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashMap;
Expand All @@ -41,39 +37,26 @@ public class IgfsModeResolver {
private final IgfsMode dfltMode;

/** Modes for particular paths. Ordered from longest to shortest. */
private ArrayList<T2<IgfsPath, IgfsMode>> modes;
private List<T2<IgfsPath, IgfsMode>> modes;

/** Cached modes per path. */
private Map<IgfsPath, IgfsMode> modesCache;

/** Cached children modes per path. */
private Map<IgfsPath, Set<IgfsMode>> childrenModesCache;

/**
* Constructor
*
* @param dfltMode Default IGFS mode.
* @param modes List of configured modes.
* @param modes List of configured modes. The order is significant as modes are added in order of occurrence.
*/
public IgfsModeResolver(IgfsMode dfltMode, @Nullable List<T2<IgfsPath, IgfsMode>> modes) {
assert dfltMode != null;

this.dfltMode = dfltMode;

if (modes != null) {
ArrayList<T2<IgfsPath, IgfsMode>> modes0 = new ArrayList<>(modes);

// Sort paths, longest first.
Collections.sort(modes0, new Comparator<Map.Entry<IgfsPath, IgfsMode>>() {
@Override public int compare(Map.Entry<IgfsPath, IgfsMode> o1,
Map.Entry<IgfsPath, IgfsMode> o2) {
return o2.getKey().components().size() - o1.getKey().components().size();
}
});

this.modes = modes0;
this.modes = modes;

if (modes != null)
modesCache = new GridBoundedConcurrentLinkedHashMap<>(MAX_PATH_CACHE);
childrenModesCache = new GridBoundedConcurrentLinkedHashMap<>(MAX_PATH_CACHE);
}
}

/**
Expand All @@ -92,7 +75,7 @@ public IgfsMode resolveMode(IgfsPath path) {

if (mode == null) {
for (T2<IgfsPath, IgfsMode> entry : modes) {
if (startsWith(path, entry.getKey())) {
if (path.isSame(entry.getKey()) || path.isSubDirectoryOf(entry.getKey())) {
// As modes ordered from most specific to least specific first mode found is ours.
mode = entry.getValue();

Expand All @@ -110,74 +93,11 @@ public IgfsMode resolveMode(IgfsPath path) {
}
}

/**
* @param path Path.
* @return Set of all modes that children paths could have.
*/
public Set<IgfsMode> resolveChildrenModes(IgfsPath path) {
assert path != null;

if (modes == null)
return Collections.singleton(dfltMode);
else {
Set<IgfsMode> children = childrenModesCache.get(path);

if (children == null) {
children = new HashSet<>(IgfsMode.values().length, 1.0f);

IgfsMode pathDefault = dfltMode;

for (T2<IgfsPath, IgfsMode> child : modes) {
if (startsWith(path, child.getKey())) {
pathDefault = child.getValue();

break;
}
else if (startsWith(child.getKey(), path))
children.add(child.getValue());
}

children.add(pathDefault);

childrenModesCache.put(path, children);
}

return children;
}
}

/**
* @return Unmodifiable copy of properly ordered modes prefixes
* or {@code null} if no modes set.
*/
@Nullable public List<T2<IgfsPath, IgfsMode>> modesOrdered() {
return modes != null ? Collections.unmodifiableList(modes) : null;
}

/**
* Check if path starts with prefix.
*
* @param path Path.
* @param prefix Prefix.
* @return {@code true} if path starts with prefix, {@code false} if not.
*/
private static boolean startsWith(IgfsPath path, IgfsPath prefix) {
List<String> p1Comps = path.components();
List<String> p2Comps = prefix.components();

if (p2Comps.size() > p1Comps.size())
return false;

for (int i = 0; i < p1Comps.size(); i++) {
if (i >= p2Comps.size() || p2Comps.get(i) == null)
// All prefix components already matched.
return true;

if (!p1Comps.get(i).equals(p2Comps.get(i)))
return false;
}

// Path and prefix components had same length and all of them matched.
return true;
}
}

0 comments on commit 9b55658

Please sign in to comment.