From 24a93840d5acd68892b0024dc058bc498036f520 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 14 Jul 2015 18:37:52 -0400 Subject: [PATCH] Add url repository whitelist Require urls for URL repository to be listed in repositories.url.allowed_urls setting. This change ensures that only authorized URLs can be accessed by elasticsearch --- .../elasticsearch/common/io/PathUtils.java | 9 ++ .../elasticsearch/common/util/URIPattern.java | 98 +++++++++++++++++++ .../org/elasticsearch/env/Environment.java | 47 +++++++++ .../repositories/uri/URLRepository.java | 62 +++++++++++- .../bwcompat/RestoreBackwardsCompatTests.java | 26 ++++- .../common/util/URIPatternTests.java | 52 ++++++++++ .../elasticsearch/env/EnvironmentTests.java | 14 +++ .../snapshots/RepositoriesTests.java | 27 ++++- .../test/rest/ElasticsearchRestTestCase.java | 2 + .../main/resources/ant/integration-tests.xml | 4 +- docs/reference/migration/migrate_2_0.asciidoc | 15 ++- docs/reference/modules/snapshots.asciidoc | 16 ++- 12 files changed, 354 insertions(+), 18 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/common/util/URIPattern.java create mode 100644 core/src/test/java/org/elasticsearch/common/util/URIPatternTests.java diff --git a/core/src/main/java/org/elasticsearch/common/io/PathUtils.java b/core/src/main/java/org/elasticsearch/common/io/PathUtils.java index 103ac19df686d..ada11bfd0faf4 100644 --- a/core/src/main/java/org/elasticsearch/common/io/PathUtils.java +++ b/core/src/main/java/org/elasticsearch/common/io/PathUtils.java @@ -92,6 +92,15 @@ public static Path get(Path[] roots, String path) { return null; } + /** + * Tries to resolve the given file uri against the list of available roots. + * + * If uri starts with one of the listed roots, it returned back by this method, otherwise null is returned. + */ + public static Path get(Path[] roots, URI uri) { + return get(roots, PathUtils.get(uri).normalize().toString()); + } + /** * Returns the default FileSystem. */ diff --git a/core/src/main/java/org/elasticsearch/common/util/URIPattern.java b/core/src/main/java/org/elasticsearch/common/util/URIPattern.java new file mode 100644 index 0000000000000..0b9bb222521a1 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/common/util/URIPattern.java @@ -0,0 +1,98 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.util; + +import org.elasticsearch.common.regex.Regex; + +import java.net.URI; +import java.net.URISyntaxException; + +/** + * URI Pattern matcher + * + * The pattern is URI in which authority, path, query and fragment can be replace with simple pattern. + * + * For example: foobar://*.local/some_path/*?*#* will match all uris with schema foobar in local domain + * with any port, with path that starts some_path and with any query and fragment. + */ +public class URIPattern { + private final URI uriPattern; + + /** + * Constructs uri pattern + * @param pattern + */ + public URIPattern(String pattern) { + try { + uriPattern = new URI(pattern); + } catch (URISyntaxException ex) { + throw new IllegalArgumentException("cannot parse URI pattern [" + pattern + "]"); + } + } + + /** + * Returns true if the given uri matches the pattern + */ + public boolean match(URI uri) { + return matchNormalized(uri.normalize()); + } + + public static boolean match(URIPattern[] patterns, URI uri) { + URI normalized = uri.normalize(); + for (URIPattern pattern : patterns) { + if (pattern.matchNormalized(normalized)) { + return true; + } + } + return false; + } + + private boolean matchNormalized(URI uri) { + if(uriPattern.isOpaque()) { + // This url only has scheme, scheme-specific part and fragment + return uri.isOpaque() && + match(uriPattern.getScheme(), uri.getScheme()) && + match(uriPattern.getSchemeSpecificPart(), uri.getSchemeSpecificPart()) && + match(uriPattern.getFragment(), uri.getFragment()); + + } else { + return match(uriPattern.getScheme(), uri.getScheme()) && + match(uriPattern.getAuthority(), uri.getAuthority()) && + match(uriPattern.getQuery(), uri.getQuery()) && + match(uriPattern.getPath(), uri.getPath()) && + match(uriPattern.getFragment(), uri.getFragment()); + } + } + + private boolean match(String pattern, String value) { + if (value == null) { + // If the pattern is empty or matches anything - it's a match + if (pattern == null || Regex.isMatchAllPattern(pattern)) { + return true; + } + } + return Regex.simpleMatch(pattern, value); + } + + @Override + public String toString() { + return uriPattern.toString(); + } +} diff --git a/core/src/main/java/org/elasticsearch/env/Environment.java b/core/src/main/java/org/elasticsearch/env/Environment.java index de426df6d0473..445ec7a61de11 100644 --- a/core/src/main/java/org/elasticsearch/env/Environment.java +++ b/core/src/main/java/org/elasticsearch/env/Environment.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.FileStore; import java.nio.file.Files; @@ -178,6 +179,52 @@ public Path resolveRepoFile(String location) { return PathUtils.get(repoFiles, location); } + /** + * Checks if the specified URL is pointing to the local file system and if it does, resolves the specified url + * against the list of configured repository roots + * + * If the specified url doesn't match any of the roots, returns null. + */ + public URL resolveRepoURL(URL url) { + try { + if ("file".equalsIgnoreCase(url.getProtocol())) { + if (url.getHost() == null || "".equals(url.getHost())) { + // only local file urls are supported + Path path = PathUtils.get(repoFiles, url.toURI()); + if (path == null) { + // Couldn't resolve against known repo locations + return null; + } + // Normalize URL + return path.toUri().toURL(); + } + return null; + } else if ("jar".equals(url.getProtocol())) { + String file = url.getFile(); + int pos = file.indexOf("!/"); + if (pos < 0) { + return null; + } + String jarTail = file.substring(pos); + String filePath = file.substring(0, pos); + URL internalUrl = new URL(filePath); + URL normalizedUrl = resolveRepoURL(internalUrl); + if (normalizedUrl == null) { + return null; + } + return new URL("jar", "", normalizedUrl.toExternalForm() + jarTail); + } else { + // It's not file or jar url and it didn't match the white list - reject + return null; + } + } catch (MalformedURLException ex) { + // cannot make sense of this file url + return null; + } catch (URISyntaxException ex) { + return null; + } + } + /** * The config location. */ diff --git a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java index c2db23917030d..f3c439b59c563 100644 --- a/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java +++ b/core/src/main/java/org/elasticsearch/repositories/uri/URLRepository.java @@ -19,12 +19,14 @@ package org.elasticsearch.repositories.uri; -import com.google.common.collect.ImmutableList; import org.elasticsearch.cluster.metadata.SnapshotId; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.url.URLBlobStore; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.URIPattern; +import org.elasticsearch.env.Environment; import org.elasticsearch.index.snapshots.IndexShardRepository; import org.elasticsearch.repositories.RepositoryException; import org.elasticsearch.repositories.RepositoryName; @@ -32,6 +34,7 @@ import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import java.io.IOException; +import java.net.URISyntaxException; import java.net.URL; import java.util.List; @@ -48,6 +51,18 @@ public class URLRepository extends BlobStoreRepository { public final static String TYPE = "url"; + public final static String[] DEFAULT_SUPPORTED_PROTOCOLS = {"http", "https", "ftp", "file", "jar"}; + + public final static String SUPPORTED_PROTOCOLS_SETTING = "repositories.url.supported_protocols"; + + public final static String ALLOWED_URLS_SETTING = "repositories.url.allowed_urls"; + + private final String[] supportedProtocols; + + private final URIPattern[] urlWhiteList; + + private final Environment environment; + private final URLBlobStore blobStore; private final BlobPath basePath; @@ -63,17 +78,25 @@ public class URLRepository extends BlobStoreRepository { * @throws IOException */ @Inject - public URLRepository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository) throws IOException { + public URLRepository(RepositoryName name, RepositorySettings repositorySettings, IndexShardRepository indexShardRepository, Environment environment) throws IOException { super(name.getName(), repositorySettings, indexShardRepository); URL url; - String path = repositorySettings.settings().get("url", settings.get("repositories.uri.url")); + String path = repositorySettings.settings().get("url", settings.get("repositories.url.url", settings.get("repositories.uri.url"))); if (path == null) { throw new RepositoryException(name.name(), "missing url"); } else { url = new URL(path); } + supportedProtocols = settings.getAsArray(SUPPORTED_PROTOCOLS_SETTING, DEFAULT_SUPPORTED_PROTOCOLS); + String[] urlWhiteList = settings.getAsArray(ALLOWED_URLS_SETTING, Strings.EMPTY_ARRAY); + this.urlWhiteList = new URIPattern[urlWhiteList.length]; + for (int i = 0; i < urlWhiteList.length; i++) { + this.urlWhiteList[i] = new URIPattern(urlWhiteList[i]); + } + this.environment = environment; listDirectories = repositorySettings.settings().getAsBoolean("list_directories", settings.getAsBoolean("repositories.uri.list_directories", true)); - blobStore = new URLBlobStore(settings, url); + URL normalizedURL = checkURL(url); + blobStore = new URLBlobStore(settings, normalizedURL); basePath = BlobPath.cleanPath(); } @@ -114,4 +137,35 @@ public void endVerification(String seed) { throw new UnsupportedOperationException("shouldn't be called"); } + /** + * Makes sure that the url is white listed or if it points to the local file system it matches one on of the root path in path.repo + */ + private URL checkURL(URL url) { + String protocol = url.getProtocol(); + if (protocol == null) { + throw new RepositoryException(repositoryName, "unknown url protocol from URL [" + url + "]"); + } + for (String supportedProtocol : supportedProtocols) { + if (supportedProtocol.equals(protocol)) { + try { + if (URIPattern.match(urlWhiteList, url.toURI())) { + // URL matches white list - no additional processing is needed + return url; + } + } catch (URISyntaxException ex) { + logger.warn("cannot parse the specified url [{}]", url); + throw new RepositoryException(repositoryName, "cannot parse the specified url [" + url + "]"); + } + // We didn't match white list - try to resolve against repo.path + URL normalizedUrl = environment.resolveRepoURL(url); + if (normalizedUrl == null) { + logger.warn("The specified url [{}] doesn't start with any repository paths specified by the path.repo setting: [{}] or by repositories.url.allowed_urls setting: [{}] ", url, environment.repoFiles()); + throw new RepositoryException(repositoryName, "file url [" + url + "] doesn't match any of the locations specified by path.repo or repositories.url.allowed_urls"); + } + return normalizedUrl; + } + } + throw new RepositoryException(repositoryName, "unsupported url protocol [" + protocol + "] from URL [" + url + "]"); + } + } diff --git a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatTests.java b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatTests.java index 999acfdda2522..a0d3dc4e10a0c 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatTests.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatTests.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.lang.reflect.Modifier; import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -59,10 +60,25 @@ public class RestoreBackwardsCompatTests extends AbstractSnapshotTests { @Override protected Settings nodeSettings(int nodeOrdinal) { - return settingsBuilder() - .put(super.nodeSettings(nodeOrdinal)) - .put("path.repo", reposRoot()) - .build(); + if (randomBoolean()) { + // Configure using path.repo + return settingsBuilder() + .put(super.nodeSettings(nodeOrdinal)) + .put("path.repo", reposRoot()) + .build(); + } else { + // Configure using url white list + try { + URI repoJarPatternUri = new URI("jar:" + reposRoot().toUri().toString() + "*.zip!/repo/"); + return settingsBuilder() + .put(super.nodeSettings(nodeOrdinal)) + .putArray("repositories.url.allowed_urls", repoJarPatternUri.toString()) + .build(); + } catch (URISyntaxException ex) { + throw new IllegalArgumentException(ex); + } + + } } @Test @@ -142,7 +158,7 @@ private List listRepoVersions(String prefix) throws Exception { private void createRepo(String prefix, String version, String repo) throws Exception { String repoFile = prefix + "-" + version + ".zip"; - URI repoFileUri = getClass().getResource(repoFile).toURI(); + URI repoFileUri = getDataPath(repoFile).toUri(); URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/"); logger.info("--> creating repository [{}] for version [{}]", repo, version); assertAcked(client().admin().cluster().preparePutRepository(repo) diff --git a/core/src/test/java/org/elasticsearch/common/util/URIPatternTests.java b/core/src/test/java/org/elasticsearch/common/util/URIPatternTests.java new file mode 100644 index 0000000000000..5c4b2a7c701bf --- /dev/null +++ b/core/src/test/java/org/elasticsearch/common/util/URIPatternTests.java @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.util; + +import org.elasticsearch.test.ElasticsearchTestCase; +import org.junit.Test; + +import java.net.URI; + +public class URIPatternTests extends ElasticsearchTestCase { + + @Test + public void testURIPattern() throws Exception { + assertTrue(new URIPattern("http://test.local/").match(new URI("http://test.local/"))); + assertFalse(new URIPattern("http://test.local/somepath").match(new URI("http://test.local/"))); + assertTrue(new URIPattern("http://test.local/somepath").match(new URI("http://test.local/somepath"))); + assertFalse(new URIPattern("http://test.local/somepath").match(new URI("http://test.local/somepath/more"))); + assertTrue(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/somepath/more"))); + assertTrue(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/somepath/more/andmore"))); + assertTrue(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/somepath/more/andmore/../bitmore"))); + assertFalse(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/somepath/../more"))); + assertFalse(new URIPattern("http://test.local/somepath/*").match(new URI("http://test.local/"))); + assertFalse(new URIPattern("http://test.local/somepath/*").match(new URI("https://test.local/somepath/more"))); + assertFalse(new URIPattern("http://test.local:1234/somepath/*").match(new URI("http://test.local/somepath/more"))); + assertFalse(new URIPattern("http://test.local:1234/somepath/*").match(new URI("http://test.local/somepath/more"))); + assertTrue(new URIPattern("http://test.local:1234/somepath/*").match(new URI("http://test.local:1234/somepath/more"))); + assertTrue(new URIPattern("http://*.local:1234/somepath/*").match(new URI("http://foobar.local:1234/somepath/more"))); + assertFalse(new URIPattern("http://*.local:1234/somepath/*").match(new URI("http://foobar.local:2345/somepath/more"))); + assertTrue(new URIPattern("http://*.local:*/somepath/*").match(new URI("http://foobar.local:2345/somepath/more"))); + assertFalse(new URIPattern("http://*.local:*/somepath/*").match(new URI("http://foobar.local:2345/somepath/more?par=val"))); + assertTrue(new URIPattern("http://*.local:*/somepath/*?*").match(new URI("http://foobar.local:2345/somepath/more?par=val"))); + assertFalse(new URIPattern("http://*.local:*/somepath/*?*").match(new URI("http://foobar.local:2345/somepath/more?par=val#frag"))); + assertTrue(new URIPattern("http://*.local:*/somepath/*?*#*").match(new URI("http://foobar.local:2345/somepath/more?par=val#frag"))); + assertTrue(new URIPattern("http://*.local/somepath/*?*#*").match(new URI("http://foobar.local/somepath/more"))); + } +} diff --git a/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java b/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java index 3eba6c46c5a21..a2ba2d3988637 100644 --- a/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java +++ b/core/src/test/java/org/elasticsearch/env/EnvironmentTests.java @@ -30,6 +30,7 @@ import java.net.URL; import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.hamcrest.CoreMatchers.endsWith; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; @@ -85,6 +86,19 @@ public void testRepositoryResolution() throws IOException { assertThat(environment.resolveRepoFile("/test/repos/../repos/repo1"), notNullValue()); assertThat(environment.resolveRepoFile("/somethingeles/repos/repo1"), nullValue()); assertThat(environment.resolveRepoFile("/test/other/repo"), notNullValue()); + + + assertThat(environment.resolveRepoURL(new URL("file:///test/repos/repo1")), notNullValue()); + assertThat(environment.resolveRepoURL(new URL("file:/test/repos/repo1")), notNullValue()); + assertThat(environment.resolveRepoURL(new URL("file://test/repos/repo1")), nullValue()); + assertThat(environment.resolveRepoURL(new URL("file:///test/repos/../repo1")), nullValue()); + assertThat(environment.resolveRepoURL(new URL("http://localhost/test/")), nullValue()); + + assertThat(environment.resolveRepoURL(new URL("jar:file:///test/repos/repo1!/repo/")), notNullValue()); + assertThat(environment.resolveRepoURL(new URL("jar:file:/test/repos/repo1!/repo/")), notNullValue()); + assertThat(environment.resolveRepoURL(new URL("jar:file:///test/repos/repo1!/repo/")).toString(), endsWith("repo1!/repo/")); + assertThat(environment.resolveRepoURL(new URL("jar:file:///test/repos/../repo1!/repo/")), nullValue()); + assertThat(environment.resolveRepoURL(new URL("jar:http://localhost/test/../repo1?blah!/repo/")), nullValue()); } } diff --git a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesTests.java b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesTests.java index f4989314f4baa..a6432e5cda5e9 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/RepositoriesTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/RepositoriesTests.java @@ -136,8 +136,9 @@ public void testMisconfiguredRepository() throws Exception { assertThat(ex.toString(), containsString("missing location")); } - logger.info("--> trying creating repository with location that is not registered in path.repo setting"); - String location = createTempDir().toAbsolutePath().toString(); + logger.info("--> trying creating fs repository with location that is not registered in path.repo setting"); + Path invalidRepoPath = createTempDir().toAbsolutePath(); + String location = invalidRepoPath.toString(); try { client().admin().cluster().preparePutRepository("test-repo") .setType("fs").setSettings(Settings.settingsBuilder().put("location", location)) @@ -146,6 +147,28 @@ public void testMisconfiguredRepository() throws Exception { } catch (RepositoryException ex) { assertThat(ex.toString(), containsString("location [" + location + "] doesn't match any of the locations specified by path.repo")); } + + String repoUrl = invalidRepoPath.toAbsolutePath().toUri().toURL().toString(); + String unsupportedUrl = repoUrl.replace("file:/", "netdoc:/"); + logger.info("--> trying creating url repository with unsupported url protocol"); + try { + client().admin().cluster().preparePutRepository("test-repo") + .setType("url").setSettings(Settings.settingsBuilder().put("url", unsupportedUrl)) + .get(); + fail("Shouldn't be here"); + } catch (RepositoryException ex) { + assertThat(ex.toString(), containsString("unsupported url protocol [netdoc]")); + } + + logger.info("--> trying creating url repository with location that is not registered in path.repo setting"); + try { + client().admin().cluster().preparePutRepository("test-repo") + .setType("url").setSettings(Settings.settingsBuilder().put("url", invalidRepoPath.toUri().toURL())) + .get(); + fail("Shouldn't be here"); + } catch (RepositoryException ex) { + assertThat(ex.toString(), containsString("doesn't match any of the locations specified by path.repo")); + } } @Test diff --git a/core/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTestCase.java b/core/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTestCase.java index b815bd98d960d..b9449d03db38f 100644 --- a/core/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTestCase.java +++ b/core/src/test/java/org/elasticsearch/test/rest/ElasticsearchRestTestCase.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.node.Node; +import org.elasticsearch.repositories.uri.URLRepository; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; import org.elasticsearch.test.rest.client.RestException; @@ -155,6 +156,7 @@ protected void afterIfFailed(List errors) { @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() + .putArray(URLRepository.ALLOWED_URLS_SETTING, "http://snapshot.test*") .put(Node.HTTP_ENABLED, true) .put(super.nodeSettings(nodeOrdinal)).build(); } diff --git a/dev-tools/src/main/resources/ant/integration-tests.xml b/dev-tools/src/main/resources/ant/integration-tests.xml index d6f8d411d5701..7835bd9629d1c 100644 --- a/dev-tools/src/main/resources/ant/integration-tests.xml +++ b/dev-tools/src/main/resources/ant/integration-tests.xml @@ -91,7 +91,7 @@ Starting up external cluster... + args="${integ.args} -Des.path.repo=${integ.repo.home} -Des.repositories.url.allowed_urls=http://snapshot.test*" /> @@ -143,7 +143,7 @@ Starting up external cluster... + args="${integ.args} -Des.path.repo=${integ.repo.home} -Des.repositories.url.allowed_urls=http://snapshot.test*"/> diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index d96cd11d4533c..946dfe0c4b813 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -548,18 +548,27 @@ As a consequence the `query` filter serves no purpose anymore and is deprecated. === Snapshot and Restore -Locations of file system repositories has to be now registered using `path.repo` setting. The `path.repo` -setting can contain one or more repository locations: +Locations of the shared file system repositories and the URL repositories with `file:` URLs has to be now registered +using `path.repo` setting. The `path.repo` setting can contain one or more repository locations: [source,yaml] --------------- path.repo: ["/mnt/daily", "/mnt/weekly"] --------------- -If the file system repository location is specified as an absolute path it has to start with one of the locations +If the repository location is specified as an absolute path it has to start with one of the locations specified in `path.repo`. If the location is specified as a relative path, it will be resolved against the first location specified in the `path.repo` setting. +URL repositories with `http:`, `https:`, and `ftp:` URLs has to be whitelisted by specifying allowed URLs in the +`repositories.url.allowed_urls` setting. This setting supports wildcards in the place of host, path, query, and +fragment. For example: + +[source,yaml] +----------------------------------- +repositories.url.allowed_urls: ["http://www.example.org/root/*", "https://*.mydomain.com/*?*#*"] +----------------------------------- + The obsolete parameters `expand_wildcards_open` and `expand_wildcards_close` are no longer supported by the snapshot and restore operations. These parameters have been replaced by a single `expand_wildcards` parameter. See <> for more. diff --git a/docs/reference/modules/snapshots.asciidoc b/docs/reference/modules/snapshots.asciidoc index 9fae7b0952dd6..2f75decfe89a9 100644 --- a/docs/reference/modules/snapshots.asciidoc +++ b/docs/reference/modules/snapshots.asciidoc @@ -118,12 +118,24 @@ The following settings are supported: ===== Read-only URL Repository The URL repository (`"type": "url"`) can be used as an alternative read-only way to access data created by the shared file -system repository. The URL specified in the `url` parameter should -point to the root of the shared filesystem repository. The following settings are supported: +system repository. The URL specified in the `url` parameter should point to the root of the shared filesystem repository. +The following settings are supported: [horizontal] `url`:: Location of the snapshots. Mandatory. +URL Repository supports the following protocols: "http", "https", "ftp", "file" and "jar". URL repositories with `http:`, +`https:`, and `ftp:` URLs has to be whitelisted by specifying allowed URLs in the `repositories.url.allowed_urls` setting. +This setting supports wildcards in the place of host, path, query, and fragment. For example: + +[source,yaml] +----------------------------------- +repositories.url.allowed_urls: ["http://www.example.org/root/*", "https://*.mydomain.com/*?*#*"] +----------------------------------- + +URL repositories with `file:` URLs can only point to locations registered in the `repo.path` setting similiar to +shared file system repository. + [float] ===== Repository plugins