FREEMARKER-148 Make usage of "DataSources" more "Freemarker" like

README.md

@@ -29,7 +29,7 @@ Currently it can be invoked as a
 Building Apache FreeMarker Generator
 -----------------------------------------------------------------------------
 
-To create the artefacts locally run
+To create the artifacts locally run
 
 > mvn clean install
 

freemarker-generator-base/README.md

@@ -4,7 +4,7 @@ Apache FreeMarker Generator Base
 This module provides common functionality for `freemarker-generator-cli` and `freemarker-generator-maven-plugin` such as
 
 * Various implementation of `javax.activation.DataSources`
-* Implementation of `DataSource` and utitity methods
+* Implementation of `DataSource` and utility methods
 * Creating `DataSources` 
 
 The code actually does not depend on Apache FreeMarker since it useful for other command line tools as well.

freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/FreeMarkerConstants.java

@@ -41,7 +41,7 @@ private FreeMarkerConstants() {
 
     /* Default group name for data sources */
     public static final String DEFAULT_GROUP = "default";
-    
+
     public static class Configuration {
 
         private Configuration() {
@@ -77,7 +77,8 @@ public static class Model {
         private Model() {
         }
 
-        public static final String DATASOURCES = "DataSources";
+        public static final String DATASOURCES = "dataSources";
+        public static final String TOOLS = "tools";
 
         public static final String FREEMARKER_CLI_ARGS = "freemarker.cli.args";
         public static final String FREEMARKER_LOCALE = "freemarker.locale";

freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java

@@ -21,7 +21,7 @@
 import org.apache.commons.io.LineIterator;
 import org.apache.freemarker.generator.base.activation.ByteArrayDataSource;
 import org.apache.freemarker.generator.base.activation.StringDataSource;
-import org.apache.freemarker.generator.base.mime.MimetypeParser;
+import org.apache.freemarker.generator.base.mime.MimeTypeParser;
 import org.apache.freemarker.generator.base.util.CloseableReaper;
 import org.apache.freemarker.generator.base.util.StringUtils;
 import org.apache.freemarker.generator.base.util.Validate;
@@ -49,9 +49,21 @@
  * There is also special support of <code>UrlDataSource</code> since
  * the content type &amp; charset might be determined using a network
  * call.
+ * <br>
+ * The implementation makes no assumption if the underlying input
+ * stream can be consumed more than once.
  */
 public class DataSource implements Closeable, javax.activation.DataSource {
 
+    public static final String METADATA_BASE_NAME = "baseName";
+    public static final String METADATA_EXTENSION = "extension";
+    public static final String METADATA_FILE_NAME = "fileName";
+    public static final String METADATA_FILE_PATH = "filePath";
+    public static final String METADATA_GROUP = "group";
+    public static final String METADATA_NAME = "name";
+    public static final String METADATA_URI = "uri";
+    public static final String METADATA_URI_PATH = "uriPath";
+
     /** Human-readable name of the data source */
     private final String name;
 
@@ -64,14 +76,14 @@ public class DataSource implements Closeable, javax.activation.DataSource {
     /** The underlying "javax.activation.DataSource" */
     private final javax.activation.DataSource dataSource;
 
-    /** Optional content type */
+    /** Content type of data source either provided by the user or fetched directly from the data source */
     private final String contentType;
 
-    /** Optional charset for directly accessing text-based content */
+    /** Charset for directly accessing text-based content */
     private final Charset charset;
 
-    /** Collect all closables handed out to the caller to be closed when the data source is closed itself */
-    private final CloseableReaper closables;
+    /** Collect all closeables handed out to the caller to be closed when the data source is closed itself */
+    private final CloseableReaper closeables;
 
     public DataSource(
             String name,
@@ -86,7 +98,7 @@ public DataSource(
         this.dataSource = requireNonNull(dataSource);
         this.contentType = contentType;
         this.charset = charset;
-        this.closables = new CloseableReaper();
+        this.closeables = new CloseableReaper();
     }
 
     @Override
@@ -111,7 +123,7 @@ public String getContentType() {
      */
     @Override
     public InputStream getInputStream() {
-        return closables.add(getUnsafeInputStream());
+        return closeables.add(getUnsafeInputStream());
     }
 
     @Override
@@ -121,32 +133,41 @@ public OutputStream getOutputStream() {
 
     @Override
     public void close() {
-        closables.close();
+        closeables.close();
     }
 
     public String getGroup() {
         return group;
     }
 
+    public String getFileName() {
+        return FilenameUtils.getName(name);
+    }
+
     public String getBaseName() {
-        return FilenameUtils.getBaseName(name);
+        return FilenameUtils.getBaseName(getFileName());
     }
 
     public String getExtension() {
-        return FilenameUtils.getExtension(name);
+        return FilenameUtils.getExtension(getFileName());
     }
 
+    /**
+     * Get the charset. If no charset can be detected UTF-8 is assumed.
+     *
+     * @return charset
+     */
     public Charset getCharset() {
-        return charset != null ? charset : MimetypeParser.getCharset(contentType(), UTF_8);
+        return charset != null ? charset : MimeTypeParser.getCharset(contentType(), UTF_8);
     }
 
     /**
-     * Get the mimetype , i.e. content type without additional charset parameter.
+     * Get the mime type , i.e. content type without additional charset parameter.
      *
-     * @return mimetype
+     * @return mime type
      */
-    public String getMimetype() {
-        return MimetypeParser.getMimetype(contentType());
+    public String getMimeType() {
+        return MimeTypeParser.getMimeType(contentType());
     }
 
     public URI getUri() {
@@ -226,8 +247,8 @@ public List<String> getLines(String charsetName) {
 
     /**
      * Returns an Iterator for the lines in an <code>InputStream</code>, using
-     * the default character encoding specified. The caller is responsible to close
-     * the line iterator.
+     * the default character encoding specified. The exposed iterator is closed
+     * by the <code>DataSource</code>.
      *
      * @return line iterator
      */
@@ -237,15 +258,16 @@ public LineIterator getLineIterator() {
 
     /**
      * Returns an Iterator for the lines in an <code>InputStream</code>, using
-     * the character encoding specified.
+     * the character encoding specified. The exposed iterator is closed
+     * by the <code>DataSource</code>.
      *
      * @param charsetName The name of the requested charset
      * @return line iterator
      */
     public LineIterator getLineIterator(String charsetName) {
         Validate.notEmpty(charsetName, "No charset name provided");
         try {
-            return closables.add(IOUtils.lineIterator(getUnsafeInputStream(), Charset.forName(charsetName)));
+            return closeables.add(IOUtils.lineIterator(getUnsafeInputStream(), Charset.forName(charsetName)));
         } catch (IOException e) {
             throw new RuntimeException("Failed to create line iterator: " + toString(), e);
         }
@@ -260,15 +282,45 @@ public byte[] getBytes() {
     }
 
     /**
-     * Matches a metadata entry with a wildcard expression.
+     * Expose various parts of the metadata as simple strings to cater for filtering in  a script.
+     *
+     * @param key key part key
+     * @return value
+     */
+    public String getMetadata(String key) {
+        Validate.notEmpty(key, "No key provided");
+        switch (key) {
+            case METADATA_BASE_NAME:
+                return getBaseName();
+            case METADATA_EXTENSION:
+                return getExtension();
+            case METADATA_FILE_NAME:
+                return getFileName();
+            case METADATA_FILE_PATH:
+                return FilenameUtils.getFullPathNoEndSeparator(uri.getPath());
+            case METADATA_GROUP:
+                return getGroup();
+            case METADATA_NAME:
+                return getName();
+            case METADATA_URI_PATH:
+                return uri.getPath();
+            case METADATA_URI:
+                return uri.toString();
+            default:
+                throw new IllegalArgumentException("Unknown key: " + key);
+        }
+    }
+
+    /**
+     * Matches a metadata key with a wildcard expression.
      *
-     * @param part     part, e.g. "name", "basename", "extension", "uri", "group"
+     * @param key      metadata key, e.g. "name", "fileName", "baseName", "extension", "uri", "group"
      * @param wildcard the wildcard string to match against
      * @return true if the wildcard expression matches
      * @see <a href="https://commons.apache.org/proper/commons-io/javadocs/api-2.7/org/apache/commons/io/FilenameUtils.html#wildcardMatch-java.lang.String-java.lang.String-">Apache Commons IO</a>
      */
-    public boolean match(String part, String wildcard) {
-        final String value = getPart(part);
+    public boolean match(String key, String wildcard) {
+        final String value = getMetadata(key);
         return FilenameUtils.wildcardMatch(value, wildcard);
     }
 
@@ -282,7 +334,7 @@ public boolean match(String part, String wildcard) {
      * @return Closable
      */
     public <T extends Closeable> T addClosable(T closeable) {
-        return closables.add(closeable);
+        return closeables.add(closeable);
     }
 
     @Override
@@ -296,7 +348,7 @@ public String toString() {
 
     /**
      * If there is no content type we ask the underlying data source. E.g. for
-     * an URL data source this information is fetched from the server.
+     * an URL data source this information is fetched from the remote server.
      *
      * @return content type
      */
@@ -307,30 +359,4 @@ private String contentType() {
             return StringUtils.firstNonEmpty(dataSource.getContentType(), MIME_APPLICATION_OCTET_STREAM);
         }
     }
-
-    private String getPart(String part) {
-        Validate.notEmpty(part, "No metadata part provided");
-        switch (part.toLowerCase()) {
-            case "basename":
-                return getBaseName();
-            case "contenttype":
-                return getContentType();
-            case "extension":
-                return getExtension();
-            case "group":
-                return getGroup();
-            case "mimetype":
-                return getContentType();
-            case "name":
-                return getName();
-            case "path":
-                return uri.getPath();
-            case "scheme":
-                return uri.getScheme();
-            case "uri":
-                return uri.toASCIIString();
-            default:
-                throw new IllegalArgumentException("Unknown part: " + part);
-        }
-    }
 }

freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java

@@ -75,13 +75,13 @@ public static DataSource fromNamedUri(NamedUri namedUri) {
         final Charset charset = getCharsetOrElse(namedUri, NO_CHARSET);
         final String mimeType = getMimeTypeOrElse(namedUri, NO_MIME_TYPE);
 
-        if (UriUtils.isHttpURI(uri)) {
-            final URL url = toURL(uri);
-            final String name = namedUri.getNameOrElse(url.getHost());
+        if (UriUtils.isHttpUri(uri)) {
+            final URL url = toUrl(uri);
+            final String name = namedUri.getNameOrElse(UriUtils.toStringWithoutFragment(uri));
             return fromUrl(name, group, url, mimeType, charset);
         } else if (UriUtils.isFileUri(uri)) {
             final File file = namedUri.getFile();
-            final String name = namedUri.getNameOrElse(file.getName());
+            final String name = namedUri.getNameOrElse(UriUtils.toStringWithoutFragment(file.toURI()));
             return fromFile(name, group, file, charset);
         } else if (UriUtils.isEnvUri(uri)) {
             // environment variables come with a leading "/" to be removed
@@ -96,39 +96,31 @@ public static DataSource fromNamedUri(NamedUri namedUri) {
         } else {
             // handle things such as "foo=some.file"
             final File file = namedUri.getFile();
-            final String name = namedUri.getNameOrElse(file.getName());
+            final String name = namedUri.getNameOrElse(UriUtils.toStringWithoutFragment(file.toURI()));
             return fromFile(name, group, file, charset);
         }
     }
 
     // == URL ===============================================================
 
-    public static DataSource fromUrl(String name, String group, URL url, Charset charset) {
-        return fromUrl(name, group, url, NO_MIME_TYPE, charset);
-    }
-
     public static DataSource fromUrl(String name, String group, URL url, String contentType, Charset charset) {
         final URLDataSource dataSource = new CachingUrlDataSource(url);
-        final URI uri = UriUtils.toURI(url);
+        final URI uri = UriUtils.toUri(url);
         return create(name, group, uri, dataSource, contentType, charset);
     }
 
     // == String ============================================================
 
-    public static DataSource fromString(String content, String contentType) {
-        return fromString(Location.STRING, DEFAULT_GROUP, content, contentType);
-    }
-
     public static DataSource fromString(String name, String group, String content, String contentType) {
         final StringDataSource dataSource = new StringDataSource(name, content, contentType, UTF_8);
-        final URI uri = UriUtils.toURI(Location.STRING, UUID.randomUUID().toString());
+        final URI uri = UriUtils.toUri(Location.STRING, UUID.randomUUID().toString());
         return create(name, group, uri, dataSource, contentType, UTF_8);
     }
 
     // == File ==============================================================
 
     public static DataSource fromFile(File file, Charset charset) {
-        return fromFile(file.getName(), DEFAULT_GROUP, file, charset);
+        return fromFile(UriUtils.toStringWithoutFragment(file.toURI()), DEFAULT_GROUP, file, charset);
     }
 
     public static DataSource fromFile(String name, String group, File file, Charset charset) {
@@ -144,15 +136,15 @@ public static DataSource fromFile(String name, String group, File file, Charset
 
     public static DataSource fromBytes(String name, String group, byte[] content, String contentType) {
         final ByteArrayDataSource dataSource = new ByteArrayDataSource(name, content);
-        final URI uri = UriUtils.toURI(Location.BYTES + ":///");
+        final URI uri = UriUtils.toUri(Location.BYTES + ":///");
         return create(name, group, uri, dataSource, contentType, UTF_8);
     }
 
     // == InputStream =======================================================
 
     public static DataSource fromInputStream(String name, String group, InputStream is, String contentType, Charset charset) {
         final InputStreamDataSource dataSource = new InputStreamDataSource(name, is);
-        final URI uri = UriUtils.toURI(Location.INPUTSTREAM + ":///");
+        final URI uri = UriUtils.toUri(Location.INPUTSTREAM + ":///");
         return create(name, group, uri, dataSource, contentType, charset);
     }
 
@@ -169,7 +161,7 @@ public static DataSource fromEnvironment(String name, String group, String conte
             final StringWriter writer = new StringWriter();
             properties.store(writer, null);
             final StringDataSource dataSource = new StringDataSource(name, writer.toString(), contentType, UTF_8);
-            final URI uri = UriUtils.toURI(Location.ENVIRONMENT, "");
+            final URI uri = UriUtils.toUri(Location.ENVIRONMENT, "");
             return create(name, group, uri, dataSource, contentType, UTF_8);
         } catch (IOException e) {
             throw new RuntimeException(e);
@@ -180,7 +172,7 @@ public static DataSource fromEnvironment(String name, String group, String key,
         Validate.notEmpty(System.getenv(key), "Environment variable not found: " + key);
 
         final StringDataSource dataSource = new StringDataSource(name, System.getenv(key), contentType, UTF_8);
-        final URI uri = UriUtils.toURI(Location.ENVIRONMENT, key);
+        final URI uri = UriUtils.toUri(Location.ENVIRONMENT, key);
         return create(name, group, uri, dataSource, contentType, UTF_8);
     }
 
@@ -225,7 +217,7 @@ private static Charset getCharsetOrElse(NamedUri namedUri, Charset def) {
         return StringUtils.isEmpty(charsetName) ? def : Charset.forName(charsetName);
     }
 
-    private static URL toURL(URI uri) {
+    private static URL toUrl(URI uri) {
         try {
             return uri.toURL();
         } catch (MalformedURLException e) {