Url repository should respect repo.path for file urls

core/src/main/java/org/elasticsearch/common/io/PathUtils.java

@@ -92,6 +92,15 @@ public static Path get(Path[] roots, String path) {
         return null;
     }
 
+    /**
+     * Tries to resolve the given file uri against the list of available roots.
+     *
+     * If uri starts with one of the listed roots, it returned back by this method, otherwise null is returned.
+     */
+    public static Path get(Path[] roots, URI uri) {
+        return get(roots, PathUtils.get(uri).normalize().toString());
+    }
+
     /**
      * Returns the default FileSystem.
      */

core/src/main/java/org/elasticsearch/common/util/URIPattern.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.util;
+
+import org.elasticsearch.common.regex.Regex;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * URI Pattern matcher
+ *
+ * The pattern is URI in which authority, path, query and fragment can be replace with simple pattern.
+ *
+ * For example: foobar://*.local/some_path/*?*#* will match all uris with schema foobar in local domain
+ * with any port, with path that starts some_path and with any query and fragment.
+ */
+public class URIPattern {
+    private final URI uriPattern;
+
+    /**
+     * Constructs uri pattern
+     * @param pattern
+     */
+    public URIPattern(String pattern) {
+        try {
+            uriPattern = new URI(pattern);
+        } catch (URISyntaxException ex) {
+            throw new IllegalArgumentException("cannot parse URI pattern [" + pattern + "]");
+        }
+    }
+
+    /**
+     * Returns true if the given uri matches the pattern
+     */
+    public boolean match(URI uri) {
+        return matchNormalized(uri.normalize());
+    }
+
+    public static boolean match(URIPattern[] patterns, URI uri) {
+        URI normalized = uri.normalize();
+        for (URIPattern pattern : patterns) {
+            if (pattern.matchNormalized(normalized)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private boolean matchNormalized(URI uri) {
+        if(uriPattern.isOpaque()) {
+            // This url only has scheme, scheme-specific part and fragment
+            return uri.isOpaque() &&
+                    match(uriPattern.getScheme(), uri.getScheme()) &&
+                    match(uriPattern.getSchemeSpecificPart(), uri.getSchemeSpecificPart()) &&
+                    match(uriPattern.getFragment(), uri.getFragment());
+
+        } else {
+            return match(uriPattern.getScheme(), uri.getScheme()) &&
+                    match(uriPattern.getAuthority(), uri.getAuthority()) &&
+                    match(uriPattern.getQuery(), uri.getQuery()) &&
+                    match(uriPattern.getPath(), uri.getPath()) &&
+                    match(uriPattern.getFragment(), uri.getFragment());
+        }
+    }
+
+    private boolean match(String pattern, String value) {
+        if (value == null) {
+            // If the pattern is empty or matches anything - it's a match
+            if (pattern == null || Regex.isMatchAllPattern(pattern)) {
+                return true;
+            }
+        }
+        return Regex.simpleMatch(pattern, value);
+    }
+
+    @Override
+    public String toString() {
+        return uriPattern.toString();
+    }
+}

core/src/main/java/org/elasticsearch/env/Environment.java

@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.net.MalformedURLException;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.FileStore;
 import java.nio.file.Files;
@@ -178,6 +179,52 @@ public Path resolveRepoFile(String location) {
         return PathUtils.get(repoFiles, location);
     }
 
+    /**
+     * Checks if the specified URL is pointing to the local file system and if it does, resolves the specified url
+     * against the list of configured repository roots
+     *
+     * If the specified url doesn't match any of the roots, returns null.
+     */
+    public URL resolveRepoURL(URL url) {
+        try {
+            if ("file".equalsIgnoreCase(url.getProtocol())) {
+                if (url.getHost() == null || "".equals(url.getHost())) {
+                    // only local file urls are supported
+                    Path path = PathUtils.get(repoFiles, url.toURI());
+                    if (path == null) {
+                        // Couldn't resolve against known repo locations
+                        return null;
+                    }
+                    // Normalize URL
+                    return path.toUri().toURL();
+                }
+                return null;
+            } else if ("jar".equals(url.getProtocol())) {
+                String file = url.getFile();
+                int pos = file.indexOf("!/");
+                if (pos < 0) {
+                    return null;
+                }
+                String jarTail = file.substring(pos);
+                String filePath = file.substring(0, pos);
+                URL internalUrl = new URL(filePath);
+                URL normalizedUrl = resolveRepoURL(internalUrl);
+                if (normalizedUrl == null) {
+                    return null;
+                }
+                return new URL("jar", "", normalizedUrl.toExternalForm() + jarTail);
+            } else {
+                // It's not file or jar url and it didn't match the white list - reject
+                return null;
+            }
+        } catch (MalformedURLException ex) {
+            // cannot make sense of this file url
+            return null;
+        } catch (URISyntaxException ex) {
+            return null;
+        }
+    }
+
     /**
      * The config location.
      */

core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java

@@ -19,19 +19,22 @@
 
 package org.elasticsearch.repositories.uri;
 
-import com.google.common.collect.ImmutableList;
 import org.elasticsearch.cluster.metadata.SnapshotId;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.blobstore.BlobPath;
 import org.elasticsearch.common.blobstore.BlobStore;
 import org.elasticsearch.common.blobstore.url.URLBlobStore;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.util.URIPattern;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.snapshots.IndexShardRepository;
 import org.elasticsearch.repositories.RepositoryException;
 import org.elasticsearch.repositories.RepositoryName;
 import org.elasticsearch.repositories.RepositorySettings;
 import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
 
 import java.io.IOException;
+import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.List;
 
@@ -48,6 +51,18 @@ public class URLRepository extends BlobStoreRepository {
 
     public final static String TYPE = "url";
 
+    public final static String[] DEFAULT_SUPPORTED_PROTOCOLS = {"http", "https", "ftp", "file", "jar"};
+
+    public final static String SUPPORTED_PROTOCOLS_SETTING = "repositories.url.supported_protocols";
+
+    public final static String ALLOWED_URLS_SETTING = "repositories.url.allowed_urls";
+
+    private final String[] supportedProtocols;
+
+    private final URIPattern[] urlWhiteList;
+
+    private final Environment environment;
+
     private final URLBlobStore blobStore;
 
     private final BlobPath basePath;
@@ -63,17 +78,25 @@ public class URLRepository extends BlobStoreRepository {
      * @throws IOException
      */
     @Inject
-    public URLRepository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository) throws IOException {
+    public URLRepository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, Environment environment) throws IOException {
         super(name.getName(), repositorySettings, indexShardRepository);
         URL url;
-        String path = repositorySettings.settings().get("url", settings.get("repositories.uri.url"));
+        String path = repositorySettings.settings().get("url", settings.get("repositories.url.url", settings.get("repositories.uri.url")));
         if (path == null) {
             throw new RepositoryException(name.name(), "missing url");
         } else {
             url = new URL(path);
         }
+        supportedProtocols = settings.getAsArray(SUPPORTED_PROTOCOLS_SETTING, DEFAULT_SUPPORTED_PROTOCOLS);
+        String[] urlWhiteList = settings.getAsArray(ALLOWED_URLS_SETTING, Strings.EMPTY_ARRAY);
+        this.urlWhiteList = new URIPattern[urlWhiteList.length];
+        for (int i = 0; i < urlWhiteList.length; i++) {
+            this.urlWhiteList[i] = new URIPattern(urlWhiteList[i]);
+        }
+        this.environment = environment;
         listDirectories = repositorySettings.settings().getAsBoolean("list_directories", settings.getAsBoolean("repositories.uri.list_directories", true));
-        blobStore = new URLBlobStore(settings, url);
+        URL normalizedURL = checkURL(url);
+        blobStore = new URLBlobStore(settings, normalizedURL);
         basePath = BlobPath.cleanPath();
     }
 
@@ -114,4 +137,35 @@ public void endVerification(String seed) {
         throw new UnsupportedOperationException("shouldn't be called");
     }
 
+    /**
+     * Makes sure that the url is white listed or if it points to the local file system it matches one on of the root path in path.repo
+     */
+    private URL checkURL(URL url) {
+        String protocol = url.getProtocol();
+        if (protocol == null) {
+            throw new RepositoryException(repositoryName, "unknown url protocol from URL [" + url + "]");
+        }
+        for (String supportedProtocol : supportedProtocols) {
+            if (supportedProtocol.equals(protocol)) {
+                try {
+                    if (URIPattern.match(urlWhiteList, url.toURI())) {
+                        // URL matches white list - no additional processing is needed
+                        return url;
+                    }
+                } catch (URISyntaxException ex) {
+                    logger.warn("cannot parse the specified url [{}]", url);
+                    throw new RepositoryException(repositoryName, "cannot parse the specified url [" + url + "]");
+                }
+                // We didn't match white list - try to resolve against repo.path
+                URL normalizedUrl = environment.resolveRepoURL(url);
+                if (normalizedUrl == null) {
+                    logger.warn("The specified url [{}] doesn't start with any repository paths specified by the path.repo setting: [{}] or by repositories.url.allowed_urls setting: [{}] ", url, environment.repoFiles());
+                    throw new RepositoryException(repositoryName, "file url [" + url + "] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls");
+                }
+                return normalizedUrl;
+            }
+        }
+        throw new RepositoryException(repositoryName, "unsupported url protocol [" + protocol + "] from URL [" + url + "]");
+    }
+
 }

core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatTests.java

@@ -40,6 +40,7 @@
 import java.io.IOException;
 import java.lang.reflect.Modifier;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -59,10 +60,25 @@ public class RestoreBackwardsCompatTests extends AbstractSnapshotTests {
 
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
-        return settingsBuilder()
-                .put(super.nodeSettings(nodeOrdinal))
-                .put("path.repo", reposRoot())
-                .build();
+        if (randomBoolean()) {
+            // Configure using path.repo
+            return settingsBuilder()
+                    .put(super.nodeSettings(nodeOrdinal))
+                    .put("path.repo", reposRoot())
+                    .build();
+        } else {
+            // Configure using url white list
+            try {
+                URI repoJarPatternUri = new URI("jar:" + reposRoot().toUri().toString() + "*.zip!/repo/");
+                return settingsBuilder()
+                        .put(super.nodeSettings(nodeOrdinal))
+                        .putArray("repositories.url.allowed_urls", repoJarPatternUri.toString())
+                        .build();
+            } catch (URISyntaxException ex) {
+                throw new IllegalArgumentException(ex);
+            }
+
+        }
     }
 
     @Test
@@ -142,7 +158,7 @@ private List<String> listRepoVersions(String prefix) throws Exception {
 
     private void createRepo(String prefix, String version, String repo) throws Exception {
         String repoFile = prefix + "-" + version + ".zip";
-        URI repoFileUri = getClass().getResource(repoFile).toURI();
+        URI repoFileUri = getDataPath(repoFile).toUri();
         URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/");
         logger.info("-->  creating repository [{}] for version [{}]", repo, version);
         assertAcked(client().admin().cluster().preparePutRepository(repo)

core/src/test/java/org/elasticsearch/common/util/URIPatternTests.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.common.util;
+
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.junit.Test;
+
+import java.net.URI;
+
+public class URIPatternTests extends ElasticsearchTestCase {
+
+    @Test
+    public void testURIPattern() throws Exception {
+        assertTrue(new URIPattern("http://test.local/").match(new URI("http://test.local/")));
+        assertFalse(new URIPattern("http://test.local/somepath").match(new URI("http://test.local/")));
+        assertTrue(new URIPattern("http://test.local/somepath").match(new URI("http://test.local/somepath")));
+        assertFalse(new URIPattern("http://test.local/somepath").match(new URI("http://test.local/somepath/more")));
+        assertTrue(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/somepath/more")));
+        assertTrue(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/somepath/more/andmore")));
+        assertTrue(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/somepath/more/andmore/../bitmore")));
+        assertFalse(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/somepath/../more")));
+        assertFalse(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/")));
+        assertFalse(new URIPattern("http://test.local/somepath/*").match(new URI("https://test.local/somepath/more")));
+        assertFalse(new URIPattern("http://test.local:1234/somepath/*").match(new URI("http://test.local/somepath/more")));
+        assertFalse(new URIPattern("http://test.local:1234/somepath/*").match(new URI("http://test.local/somepath/more")));
+        assertTrue(new URIPattern("http://test.local:1234/somepath/*").match(new URI("http://test.local:1234/somepath/more")));
+        assertTrue(new URIPattern("http://*.local:1234/somepath/*").match(new URI("http://foobar.local:1234/somepath/more")));
+        assertFalse(new URIPattern("http://*.local:1234/somepath/*").match(new URI("http://foobar.local:2345/somepath/more")));
+        assertTrue(new URIPattern("http://*.local:*/somepath/*").match(new URI("http://foobar.local:2345/somepath/more")));
+        assertFalse(new URIPattern("http://*.local:*/somepath/*").match(new URI("http://foobar.local:2345/somepath/more?par=val")));
+        assertTrue(new URIPattern("http://*.local:*/somepath/*?*").match(new URI("http://foobar.local:2345/somepath/more?par=val")));
+        assertFalse(new URIPattern("http://*.local:*/somepath/*?*").match(new URI("http://foobar.local:2345/somepath/more?par=val#frag")));
+        assertTrue(new URIPattern("http://*.local:*/somepath/*?*#*").match(new URI("http://foobar.local:2345/somepath/more?par=val#frag")));
+        assertTrue(new URIPattern("http://*.local/somepath/*?*#*").match(new URI("http://foobar.local/somepath/more")));
+    }
+}