Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support N5Url attribute paths for Translated N5Reader/Writer #46

Draft
wants to merge 6 commits into
base: master
Choose a base branch
from
Draft

Support N5Url attribute paths for Translated N5Reader/Writer #46

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3483fda
feat: support ContainerMetadataNode which extends AbstractGsonParser
cmhulbert Dec 21, 2022
0c4a498
feat: extend AbstractGsonParser for attribute path support
cmhulbert Dec 21, 2022
1b10826
feat: getOrCreateGroup when setting attributes
cmhulbert Dec 21, 2022
6482904
refactor: delegate to ContainerMetadataNode
cmhulbert Dec 21, 2022
90236b8
feat(test): support createN5Writer with location
cmhulbert Dec 21, 2022
3980893
build: use snapshot version until released
cmhulbert Dec 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@
<!-- NB: Deploy releases to the SciJava Maven repository. -->
<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>

<n5.version>2.5.1</n5.version>
<n5.version>2.5.2-SNAPSHOT</n5.version>
<bigdataviewer-core-test.version>5.1.0</bigdataviewer-core-test.version>
<bigdataviewer-vistools-test.version>1.0.0-beta-11</bigdataviewer-vistools-test.version>
<alphanumeric-comparator.version>1.4.1</alphanumeric-comparator.version>
Expand Down
228 changes: 97 additions & 131 deletions src/main/java/org/janelia/saalfeldlab/n5/container/ContainerMetadataNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,23 @@
package org.janelia.saalfeldlab.n5.container;

import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;
import org.janelia.saalfeldlab.n5.AbstractGsonReader;
import org.janelia.saalfeldlab.n5.DataBlock;
import org.janelia.saalfeldlab.n5.DatasetAttributes;
import org.janelia.saalfeldlab.n5.GsonAttributesParser;
import org.janelia.saalfeldlab.n5.N5Reader;
import org.janelia.saalfeldlab.n5.N5TreeNode;
import org.janelia.saalfeldlab.n5.N5URL;
import org.janelia.saalfeldlab.n5.N5Writer;
import org.janelia.saalfeldlab.n5.translation.JqUtils;

import java.io.IOException;
import java.lang.reflect.Array;
import java.lang.reflect.Type;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -15,58 +28,55 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.janelia.saalfeldlab.n5.DataBlock;
import org.janelia.saalfeldlab.n5.DatasetAttributes;
import org.janelia.saalfeldlab.n5.GsonAttributesParser;
import org.janelia.saalfeldlab.n5.N5Reader;
import org.janelia.saalfeldlab.n5.N5TreeNode;
import org.janelia.saalfeldlab.n5.N5Writer;
import org.janelia.saalfeldlab.n5.translation.JqUtils;

import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSyntaxException;
import com.google.gson.reflect.TypeToken;

/**
*
*
* @author John Bogovic
*/

public class ContainerMetadataNode implements N5Writer {
protected HashMap<String, JsonElement> attributes;
public class ContainerMetadataNode extends AbstractGsonReader implements N5Writer {
protected JsonElement attributes;
protected Map<String, ContainerMetadataNode> children;
protected final transient Gson gson;

public ContainerMetadataNode() {
gson = JqUtils.buildGson(null);
attributes = new HashMap<String, JsonElement>();
super(JqUtils.gsonBuilder(null));
attributes = new JsonObject();
children = new HashMap<String, ContainerMetadataNode >();
addPathsRecursive();
}

public ContainerMetadataNode(final HashMap<String, JsonElement> attributes,
public ContainerMetadataNode(final JsonElement attributes,
final Map<String, ContainerMetadataNode> children, final Gson gson) {
super(JqUtils.newBuilder(gson));
this.attributes = attributes;
this.children = children;
this.gson = gson;
}

public ContainerMetadataNode( ContainerMetadataNode other) {
gson = other.gson;
super(JqUtils.newBuilder(other.gson));
attributes = other.attributes;
children = other.children;
}

public HashMap<String, JsonElement> getAttributes() {
if (!attributes.isJsonObject()) return new HashMap<>();
final Type mapType = new TypeToken<HashMap<String, JsonElement>>(){}.getType();
return gson.fromJson(attributes, mapType);
}
public JsonElement getAttributesJson() {
return attributes;
}

public HashMap<String, JsonElement> getAttributes( String pathName ) {
return getNode( pathName ).map( x -> x.getAttributes() ).orElse( new HashMap<>());

public void setAttributesJson(JsonElement json) {
attributes = json;
}

@Override public HashMap<String, JsonElement> getAttributes( String pathName ) {
return getNode( pathName ).map(ContainerMetadataNode::getAttributes).orElse( new HashMap<>());
}

@Override public JsonElement getAttributesJson(String pathName) {

return getNode( pathName ).map(ContainerMetadataNode::getAttributesJson).orElse( new JsonObject());
}

public Map<String, ContainerMetadataNode> getChildren() {
Expand Down Expand Up @@ -115,15 +125,15 @@ public Stream<ContainerMetadataNode> pathToChild(final String path) {
* @return the path to this node from the root.
*/
public String getPath() {
if( attributes.containsKey("path"))
return attributes.get("path").getAsString();
if( attributes.isJsonObject() && attributes.getAsJsonObject().has("path"))
return attributes.getAsJsonObject().get("path").getAsString();
else
return "";
}

public Stream<String> getChildPathsRecursive( String thisPath ) {
return Stream.concat( Stream.of( thisPath ),
this.children.keySet().stream().flatMap( k ->
this.children.keySet().stream().flatMap( k ->
this.children.get(k).getChildPathsRecursive( thisPath + "/" + k )));
}

Expand All @@ -140,14 +150,17 @@ public void addPathsRecursive() {
* @param thisPath path to a node
*/
public void addPathsRecursive( String thisPath ) {
attributes.put("path", new JsonPrimitive( thisPath ));
if (attributes.isJsonObject()) {
attributes.getAsJsonObject().addProperty("path", thisPath );
}
for ( String childPath : children.keySet() )
children.get(childPath).addPathsRecursive( thisPath + "/" + childPath );
}

public Optional<ContainerMetadataNode> getParent( final String path ) {
final String groupSeparator = "/";
final String normPath = path.replaceAll("^(" + groupSeparator + "*)|(" + groupSeparator + "*$)", "");
if (!normPath.contains(groupSeparator)) return Optional.empty();
final String parentPath = normPath.substring(0, normPath.lastIndexOf( groupSeparator ) );
return getNode( parentPath );
}
Expand All @@ -165,6 +178,18 @@ public Optional<ContainerMetadataNode> getNode( final String path ) {
return Optional.empty();
}

public ContainerMetadataNode getOrCreateGroup(String pathName) {
Optional<ContainerMetadataNode> childOpt = getNode(pathName);
if (!childOpt.isPresent()) {
createGroup(pathName);
childOpt = getNode(pathName);
}

/* We create the group if it doesn't exist, so it should always exist at this point. */
//noinspection OptionalGetWithoutIsPresent
return childOpt.get();
}

public Optional<ContainerMetadataNode> getChild(final String relativePath ) {
return getChild(relativePath, "/");
}
Expand Down Expand Up @@ -205,37 +230,6 @@ public Optional<ContainerMetadataNode> getChild(final String relativePath, final
return c.getChild(relToChild, groupSeparator);
}

@Override
public <T> T getAttribute( String path, String key, Class<T> clazz ) {
return getNode( path ).map( x -> gson.fromJson( x.getAttributes().get(key), clazz))
.orElseGet( () -> null );
}

// @Override
// public <T> T getAttribute(String pathName, String key, Class<T> clazz) {
//
// Optional<ContainerMetadataNode> nodeOpt = getNode(pathName);
// if( nodeOpt.isPresent() ) {
// return gson.fromJson( nodeOpt.get().getAttributes().get(key), clazz );
// }
// return null;
// }

@Override
public <T> T getAttribute(String pathName, String key, Type type) throws IOException {
Optional<ContainerMetadataNode> nodeOpt = getNode(pathName);
if( nodeOpt.isPresent() ) {
return gson.fromJson( nodeOpt.get().getAttributes().get(key), type );
}
return null;
}

@Override
public DatasetAttributes getDatasetAttributes(String pathName) throws IOException {
// TODO Auto-generated method stub
return null;
}

@Override
public boolean exists(String pathName) {
return getNode( pathName ).isPresent();
Expand All @@ -252,63 +246,28 @@ public String[] list(String pathName) throws IOException {
return new String[]{};
}

@Override
public Map<String, Class<?>> listAttributes(String pathName) throws IOException {
final HashMap<String, JsonElement> jsonElementMap = getAttributes(pathName);
final HashMap<String, Class<?>> attributes = new HashMap<>();
jsonElementMap.forEach(
(key, jsonElement) -> {
final Class<?> clazz;
if (jsonElement.isJsonNull())
clazz = null;
else if (jsonElement.isJsonPrimitive())
clazz = GsonAttributesParser.classForJsonPrimitive((JsonPrimitive)jsonElement);
else if (jsonElement.isJsonArray()) {
final JsonArray jsonArray = (JsonArray)jsonElement;
Class<?> arrayElementClass = Object.class;
if (jsonArray.size() > 0) {
final JsonElement firstElement = jsonArray.get(0);
if (firstElement.isJsonPrimitive()) {
arrayElementClass = GsonAttributesParser.classForJsonPrimitive(firstElement.getAsJsonPrimitive());
for (int i = 1; i < jsonArray.size() && arrayElementClass != Object.class; ++i) {
final JsonElement element = jsonArray.get(i);
if (element.isJsonPrimitive()) {
final Class<?> nextArrayElementClass = GsonAttributesParser.classForJsonPrimitive(element.getAsJsonPrimitive());
if (nextArrayElementClass != arrayElementClass)
if (nextArrayElementClass == double.class && arrayElementClass == long.class)
arrayElementClass = double.class;
else {
arrayElementClass = Object.class;
break;
}
} else {
arrayElementClass = Object.class;
break;
}
}
}
clazz = Array.newInstance(arrayElementClass, 0).getClass();
} else
clazz = Object[].class;
}
else
clazz = Object.class;
attributes.put(key, clazz);
});
return attributes;
}

@Override
public <T> void setAttribute( final String pathName, final String key, final T attribute) {
setAttributes(pathName, Collections.singletonMap(key, attribute));
final ContainerMetadataNode node = getOrCreateGroup(pathName);

node.attributes = GsonAttributesParser.insertAttribute(
node.attributes,
N5URL.normalizeAttributePath(key),
attribute,
gson
);
}

@Override
public void setAttributes(String pathName, Map<String, ?> attributes) {
final Type mapType = new TypeToken<HashMap<String, JsonElement>>(){}.getType();
JsonElement json = gson.toJsonTree(attributes);
HashMap<String, JsonElement> map = gson.fromJson(json, mapType);
getNode( pathName ).ifPresent( x -> x.attributes.putAll(map) );
final ContainerMetadataNode node = getOrCreateGroup(pathName);

/* If we are setting attributes via map on a non-json-object, we overwrite the existing jsonElement*/
if (!node.attributes.isJsonObject()) {
node.setAttributesJson(new JsonObject());
}
final JsonObject currentAttributes = node.attributes.getAsJsonObject();
attributes.forEach((key, value) -> currentAttributes.add(key, gson.toJsonTree(value)));
}

@Override
Expand Down Expand Up @@ -352,7 +311,7 @@ public boolean remove(String pathName) {
if( exists(pathName))
{
final String normPath = pathName.replaceAll("^(" + groupSeparator + "*)|(" + groupSeparator + "*$)", "");
if( normPath.isEmpty())
if( normPath.isEmpty())
return remove();
else {
final String name = pathName.substring( pathName.lastIndexOf("/") + 1 );
Expand All @@ -366,7 +325,7 @@ public boolean remove(String pathName) {

@Override
public boolean remove() {
attributes.clear();
attributes = new JsonObject();
children.clear();
return true;
}
Expand Down Expand Up @@ -431,21 +390,20 @@ public static <N extends GsonAttributesParser & N5Reader > ContainerMetadataNode

public static <N extends GsonAttributesParser & N5Reader> ContainerMetadataNode buildHelper(final N n5, N5TreeNode baseNode ) {

HashMap<String, JsonElement> attrs = null;
JsonElement attrs;
try {
attrs = n5.getAttributes(baseNode.getPath());
} catch (IOException e) { }
attrs = n5.getAttributesJson(baseNode.getPath());
} catch (IOException e) {
attrs = new JsonObject();
}

final List<N5TreeNode> children = baseNode.childrenList();

final HashMap<String, ContainerMetadataNode> childMap = new HashMap<>();
for (N5TreeNode child : children)
childMap.put(child.getNodeName(), buildHelper(n5, child));

if ( attrs != null )
return new ContainerMetadataNode(attrs, childMap, n5.getGson());
else
return new ContainerMetadataNode(new HashMap<>(), childMap, n5.getGson());
return new ContainerMetadataNode(attrs, childMap, n5.getGson());
}

public static <T extends N5Reader> ContainerMetadataNode buildN5(final T n5, final String dataset, final Gson gson )
Expand All @@ -467,17 +425,25 @@ public static <T extends N5Reader> ContainerMetadataNode buildN5(final T n5, fin
}

public static ContainerMetadataNode buildHelperN5(final N5Reader n5, N5TreeNode baseNode, Gson gson ) {
final Optional<HashMap<String, JsonElement>> attrs = getMetadataMapN5(n5, baseNode.getPath(), gson );
final List<N5TreeNode> children = baseNode.childrenList();
JsonElement attributeElement = new JsonObject();
if (n5 instanceof GsonAttributesParser) {
try {
attributeElement = ((GsonAttributesParser)n5).getAttributesJson(baseNode.getPath());
} catch (IOException e) {
}
} else {
final Optional<HashMap<String, JsonElement>> attrs = getMetadataMapN5(n5, baseNode.getPath(), gson );
if (attrs.isPresent()) {
attrs.get().forEach(attributeElement.getAsJsonObject()::add);
}
}

final List<N5TreeNode> children = baseNode.childrenList();
final HashMap<String, ContainerMetadataNode> childMap = new HashMap<>();
for (N5TreeNode child : children)
childMap.put(child.getNodeName(), buildHelperN5(n5, child, gson));

if (attrs.isPresent())
return new ContainerMetadataNode(attrs.get(), childMap, gson );
else
return new ContainerMetadataNode(new HashMap<>(), childMap, gson);
return new ContainerMetadataNode(attributeElement, childMap, gson );
}

public static Optional<HashMap<String, JsonElement>> getMetadataMapN5(final N5Reader n5, final String dataset,
Expand Down
Loading