From cb27419c5a4b3be5840171ccde50921e8acef6e7 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Wed, 14 Jun 2023 09:40:46 +0100 Subject: [PATCH 1/5] Convert NodeMetadata.oldestIndexVersion to IndexVersion --- .../cluster/metadata/Metadata.java | 14 +++++------ .../elasticsearch/env/NodeEnvironment.java | 3 ++- .../org/elasticsearch/env/NodeMetadata.java | 19 ++++++++------- .../gateway/PersistedClusterStateService.java | 17 ++++++------- .../elasticsearch/env/NodeMetadataTests.java | 24 ++++++++++++------- ...tyImplicitBehaviorBootstrapCheckTests.java | 11 +++++---- .../xpack/security/SecurityTests.java | 3 ++- 7 files changed, 51 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 99b075754a008..8b8a2f7078166 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.CollectionUtil; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.Diffable; @@ -50,6 +49,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.transport.Transports; @@ -231,7 +231,7 @@ default boolean isRestorable() { private volatile SortedMap indicesLookup; private final Map mappingsByHash; - private final Version oldestIndexVersion; + private final IndexVersion oldestIndexVersion; private Metadata( String clusterUUID, @@ -256,7 +256,7 @@ private Metadata( String[] visibleClosedIndices, SortedMap indicesLookup, Map mappingsByHash, - Version oldestIndexVersion, + IndexVersion oldestIndexVersion, Map reservedStateMetadata ) { this.clusterUUID = clusterUUID; @@ -638,7 +638,7 @@ public Metadata withAddedIndex(IndexMetadata index) { updatedVisibleClosedIndices, null, updatedMappingsByHash, - index.getCompatibilityVersion().before(oldestIndexVersion) ? index.getCompatibilityVersion() : oldestIndexVersion, + IndexVersion.min(IndexVersion.fromId(index.getCompatibilityVersion().id), oldestIndexVersion), reservedStateMetadata ); } @@ -719,7 +719,7 @@ public CoordinationMetadata coordinationMetadata() { return this.coordinationMetadata; } - public Version oldestIndexVersion() { + public IndexVersion oldestIndexVersion() { return this.oldestIndexVersion; } @@ -2272,7 +2272,7 @@ public Metadata build(boolean skipNameCollisionChecks) { final List visibleClosedIndices = new ArrayList<>(); final ImmutableOpenMap indicesMap = indices.build(); - int oldestIndexVersionId = Version.CURRENT.id; + int oldestIndexVersionId = IndexVersion.CURRENT.id(); int totalNumberOfShards = 0; int totalOpenIndexShards = 0; @@ -2362,7 +2362,7 @@ public Metadata build(boolean skipNameCollisionChecks) { visibleClosedIndicesArray, indicesLookup, Collections.unmodifiableMap(mappingsByHash), - Version.fromId(oldestIndexVersionId), + IndexVersion.fromId(oldestIndexVersionId), Collections.unmodifiableMap(reservedStateMetadata) ); } diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 7d3e5fcfccfe3..61f05af53e803 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -47,6 +47,7 @@ import org.elasticsearch.gateway.PersistedClusterStateService; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.FsDirectoryFactory; @@ -623,7 +624,7 @@ private static NodeMetadata loadNodeMetadata(Settings settings, Logger logger, D assert nodeIds.isEmpty() : nodeIds; // If we couldn't find legacy metadata, we set the latest index version to this version. This happens // when we are starting a new node and there are no indices to worry about. - metadata = new NodeMetadata(generateNodeId(settings), Version.CURRENT, Version.CURRENT); + metadata = new NodeMetadata(generateNodeId(settings), Version.CURRENT, IndexVersion.CURRENT); } else { assert nodeIds.equals(Collections.singleton(legacyMetadata.nodeId())) : nodeIds + " doesn't match " + legacyMetadata; metadata = legacyMetadata; diff --git a/server/src/main/java/org/elasticsearch/env/NodeMetadata.java b/server/src/main/java/org/elasticsearch/env/NodeMetadata.java index e77e9b0709d7e..1de3539bf93ed 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeMetadata.java +++ b/server/src/main/java/org/elasticsearch/env/NodeMetadata.java @@ -10,6 +10,7 @@ import org.elasticsearch.Version; import org.elasticsearch.gateway.MetadataStateFormat; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; @@ -36,13 +37,13 @@ public final class NodeMetadata { private final Version previousNodeVersion; - private final Version oldestIndexVersion; + private final IndexVersion oldestIndexVersion; private NodeMetadata( final String nodeId, final Version nodeVersion, final Version previousNodeVersion, - final Version oldestIndexVersion + final IndexVersion oldestIndexVersion ) { this.nodeId = Objects.requireNonNull(nodeId); this.nodeVersion = Objects.requireNonNull(nodeVersion); @@ -50,7 +51,7 @@ private NodeMetadata( this.oldestIndexVersion = Objects.requireNonNull(oldestIndexVersion); } - public NodeMetadata(final String nodeId, final Version nodeVersion, final Version oldestIndexVersion) { + public NodeMetadata(final String nodeId, final Version nodeVersion, final IndexVersion oldestIndexVersion) { this(nodeId, nodeVersion, nodeVersion, oldestIndexVersion); } @@ -103,7 +104,7 @@ public Version previousNodeVersion() { return previousNodeVersion; } - public Version oldestIndexVersion() { + public IndexVersion oldestIndexVersion() { return oldestIndexVersion; } @@ -141,7 +142,7 @@ private static class Builder { String nodeId; Version nodeVersion; Version previousNodeVersion; - Version oldestIndexVersion; + IndexVersion oldestIndexVersion; public void setNodeId(String nodeId) { this.nodeId = nodeId; @@ -156,12 +157,12 @@ public void setPreviousNodeVersionId(int previousNodeVersionId) { } public void setOldestIndexVersion(int oldestIndexVersion) { - this.oldestIndexVersion = Version.fromId(oldestIndexVersion); + this.oldestIndexVersion = IndexVersion.fromId(oldestIndexVersion); } public NodeMetadata build() { final Version nodeVersion; - final Version oldestIndexVersion; + final IndexVersion oldestIndexVersion; if (this.nodeVersion == null) { assert Version.CURRENT.major <= Version.V_7_0_0.major + 1 : "version is required in the node metadata from v9 onwards"; nodeVersion = Version.V_EMPTY; @@ -172,7 +173,7 @@ public NodeMetadata build() { previousNodeVersion = nodeVersion; } if (this.oldestIndexVersion == null) { - oldestIndexVersion = Version.V_EMPTY; + oldestIndexVersion = IndexVersion.ZERO; } else { oldestIndexVersion = this.oldestIndexVersion; } @@ -208,7 +209,7 @@ protected XContentBuilder newXContentBuilder(XContentType type, OutputStream str public void toXContent(XContentBuilder builder, NodeMetadata nodeMetadata) throws IOException { builder.field(NODE_ID_KEY, nodeMetadata.nodeId); builder.field(NODE_VERSION_KEY, nodeMetadata.nodeVersion.id); - builder.field(OLDEST_INDEX_VERSION_KEY, nodeMetadata.oldestIndexVersion.id); + builder.field(OLDEST_INDEX_VERSION_KEY, nodeMetadata.oldestIndexVersion.id()); } @Override diff --git a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java index 290360a4be211..3f9b55de7b5b1 100644 --- a/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/PersistedClusterStateService.java @@ -70,6 +70,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeMetadata; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -343,7 +344,7 @@ public record OnDiskStateMetadata( public static NodeMetadata nodeMetadata(Path... dataPaths) throws IOException { String nodeId = null; Version version = null; - Version oldestIndexVersion = Version.V_EMPTY; + IndexVersion oldestIndexVersion = IndexVersion.ZERO; for (final Path dataPath : dataPaths) { final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME); if (Files.exists(indexPath)) { @@ -361,9 +362,9 @@ public static NodeMetadata nodeMetadata(Path... dataPaths) throws IOException { nodeId = thisNodeId; version = Version.fromId(Integer.parseInt(userData.get(NODE_VERSION_KEY))); if (userData.containsKey(OLDEST_INDEX_VERSION_KEY)) { - oldestIndexVersion = Version.fromId(Integer.parseInt(userData.get(OLDEST_INDEX_VERSION_KEY))); + oldestIndexVersion = IndexVersion.fromId(Integer.parseInt(userData.get(OLDEST_INDEX_VERSION_KEY))); } else { - oldestIndexVersion = Version.V_EMPTY; + oldestIndexVersion = IndexVersion.ZERO; } } } catch (IndexNotFoundException e) { @@ -835,7 +836,7 @@ void prepareCommit( String nodeId, long currentTerm, long lastAcceptedVersion, - Version oldestIndexVersion, + IndexVersion oldestIndexVersion, String clusterUUID, boolean clusterUUIDCommitted ) throws IOException { @@ -846,7 +847,7 @@ void prepareCommit( commitData.put(CURRENT_TERM_KEY, Long.toString(currentTerm)); commitData.put(LAST_ACCEPTED_VERSION_KEY, Long.toString(lastAcceptedVersion)); commitData.put(NODE_VERSION_KEY, Integer.toString(Version.CURRENT.id)); - commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldestIndexVersion.id)); + commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldestIndexVersion.id())); commitData.put(NODE_ID_KEY, nodeId); commitData.put(CLUSTER_UUID_KEY, clusterUUID); commitData.put(CLUSTER_UUID_COMMITTED_KEY, Boolean.toString(clusterUUIDCommitted)); @@ -1211,7 +1212,7 @@ private WriterStats addMetadata(Metadata metadata) throws IOException { public void writeIncrementalTermUpdateAndCommit( long currentTerm, long lastAcceptedVersion, - Version oldestIndexVersion, + IndexVersion oldestIndexVersion, String clusterUUID, boolean clusterUUIDCommitted ) throws IOException { @@ -1223,7 +1224,7 @@ public void writeIncrementalTermUpdateAndCommit( void commit( long currentTerm, long lastAcceptedVersion, - Version oldestIndexVersion, + IndexVersion oldestIndexVersion, String clusterUUID, boolean clusterUUIDCommitted ) throws IOException { @@ -1251,7 +1252,7 @@ private boolean assertOnCommit() { private void prepareCommit( long currentTerm, long lastAcceptedVersion, - Version oldestIndexVersion, + IndexVersion oldestIndexVersion, String clusterUUID, boolean clusterUUIDCommitted ) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java index 5f140e5003f1f..dfaa4e52c0e30 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java @@ -10,9 +10,11 @@ import org.elasticsearch.Version; import org.elasticsearch.core.Tuple; import org.elasticsearch.gateway.MetadataStateFormat; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.EqualsHashCodeTestUtils; import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.test.index.IndexVersionUtils; import java.io.IOException; import java.io.InputStream; @@ -26,16 +28,20 @@ import static org.hamcrest.Matchers.startsWith; public class NodeMetadataTests extends ESTestCase { + // (Index)VersionUtils.randomVersion() only returns known versions, which are necessarily no later than (Index)Version.CURRENT; + // however we want to also consider our behaviour with all versions, so occasionally pick up a truly random version. private Version randomVersion() { - // VersionUtils.randomVersion() only returns known versions, which are necessarily no later than Version.CURRENT; however we want - // also to consider our behaviour with all versions, so occasionally pick up a truly random version. return rarely() ? Version.fromId(randomInt()) : VersionUtils.randomVersion(random()); } + private IndexVersion randomIndexVersion() { + return rarely() ? IndexVersion.fromId(randomInt()) : IndexVersionUtils.randomVersion(random()); + } + public void testEqualsHashcodeSerialization() { final Path tempDir = createTempDir(); EqualsHashCodeTestUtils.checkEqualsAndHashCode( - new NodeMetadata(randomAlphaOfLength(10), randomVersion(), randomVersion()), + new NodeMetadata(randomAlphaOfLength(10), randomVersion(), randomIndexVersion()), nodeMetadata -> { final long generation = NodeMetadata.FORMAT.writeAndCleanup(nodeMetadata, tempDir); final Tuple nodeMetadataLongTuple = NodeMetadata.FORMAT.loadLatestStateWithGeneration( @@ -60,7 +66,7 @@ public void testEqualsHashcodeSerialization() { default -> new NodeMetadata( nodeMetadata.nodeId(), nodeMetadata.nodeVersion(), - randomValueOtherThan(nodeMetadata.oldestIndexVersion(), this::randomVersion) + randomValueOtherThan(nodeMetadata.oldestIndexVersion(), this::randomIndexVersion) ); } ); @@ -90,7 +96,7 @@ public void testUpgradesLegitimateVersions() { v -> v.after(Version.CURRENT) || v.before(Version.CURRENT.minimumCompatibilityVersion()), this::randomVersion ), - Version.CURRENT + IndexVersion.CURRENT ).upgradeToCurrentVersion(); assertThat(nodeMetadata.nodeVersion(), equalTo(Version.CURRENT)); assertThat(nodeMetadata.nodeId(), equalTo(nodeId)); @@ -101,7 +107,7 @@ public void testUpgradesMissingVersion() { final IllegalStateException illegalStateException = expectThrows( IllegalStateException.class, - () -> new NodeMetadata(nodeId, Version.V_EMPTY, Version.CURRENT).upgradeToCurrentVersion() + () -> new NodeMetadata(nodeId, Version.V_EMPTY, IndexVersion.CURRENT).upgradeToCurrentVersion() ); assertThat( illegalStateException.getMessage(), @@ -112,7 +118,7 @@ public void testUpgradesMissingVersion() { public void testDoesNotUpgradeFutureVersion() { final IllegalStateException illegalStateException = expectThrows( IllegalStateException.class, - () -> new NodeMetadata(randomAlphaOfLength(10), tooNewVersion(), Version.CURRENT).upgradeToCurrentVersion() + () -> new NodeMetadata(randomAlphaOfLength(10), tooNewVersion(), IndexVersion.CURRENT).upgradeToCurrentVersion() ); assertThat( illegalStateException.getMessage(), @@ -123,7 +129,7 @@ public void testDoesNotUpgradeFutureVersion() { public void testDoesNotUpgradeAncientVersion() { final IllegalStateException illegalStateException = expectThrows( IllegalStateException.class, - () -> new NodeMetadata(randomAlphaOfLength(10), tooOldVersion(), Version.CURRENT).upgradeToCurrentVersion() + () -> new NodeMetadata(randomAlphaOfLength(10), tooOldVersion(), IndexVersion.CURRENT).upgradeToCurrentVersion() ); assertThat( illegalStateException.getMessage(), @@ -144,7 +150,7 @@ public void testUpgradeMarksPreviousVersion() { final String nodeId = randomAlphaOfLength(10); final Version version = VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.V_8_0_0); - final NodeMetadata nodeMetadata = new NodeMetadata(nodeId, version, Version.CURRENT).upgradeToCurrentVersion(); + final NodeMetadata nodeMetadata = new NodeMetadata(nodeId, version, IndexVersion.CURRENT).upgradeToCurrentVersion(); assertThat(nodeMetadata.nodeVersion(), equalTo(Version.CURRENT)); assertThat(nodeMetadata.previousNodeVersion(), equalTo(version)); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheckTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheckTests.java index 4297fecdb01a5..27796b5ad37aa 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheckTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityImplicitBehaviorBootstrapCheckTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; import org.elasticsearch.env.NodeMetadata; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.license.ClusterStateLicenseService; import org.elasticsearch.license.License; import org.elasticsearch.license.LicensesMetadata; @@ -33,7 +34,7 @@ public void testFailureUpgradeFrom7xWithImplicitSecuritySettings() throws Except Version.V_8_0_0, () -> VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.V_8_0_0) ); - NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, Version.CURRENT); + NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, IndexVersion.CURRENT); nodeMetadata = nodeMetadata.upgradeToCurrentVersion(); ClusterStateLicenseService licenseService = mock(ClusterStateLicenseService.class); Metadata metadata = createLicensesMetadata(previousVersion, randomFrom("basic", "trial")); @@ -67,7 +68,7 @@ public void testUpgradeFrom7xWithImplicitSecuritySettingsOnGoldPlus() throws Exc Version.V_8_0_0, () -> VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.V_8_0_0) ); - NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, Version.CURRENT); + NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, IndexVersion.CURRENT); nodeMetadata = nodeMetadata.upgradeToCurrentVersion(); ClusterStateLicenseService licenseService = mock(ClusterStateLicenseService.class); Metadata metadata = createLicensesMetadata(previousVersion, randomFrom("gold", "platinum")); @@ -85,7 +86,7 @@ public void testUpgradeFrom7xWithExplicitSecuritySettings() throws Exception { Version.V_8_0_0, () -> VersionUtils.randomVersionBetween(random(), Version.CURRENT.minimumCompatibilityVersion(), Version.V_8_0_0) ); - NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, Version.CURRENT); + NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, IndexVersion.CURRENT); nodeMetadata = nodeMetadata.upgradeToCurrentVersion(); ClusterStateLicenseService licenseService = mock(ClusterStateLicenseService.class); BootstrapCheck.BootstrapCheckResult result = new SecurityImplicitBehaviorBootstrapCheck(nodeMetadata, licenseService).check( @@ -99,7 +100,7 @@ public void testUpgradeFrom7xWithExplicitSecuritySettings() throws Exception { public void testUpgradeFrom8xWithImplicitSecuritySettings() throws Exception { final Version previousVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, null); - NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, Version.CURRENT); + NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, IndexVersion.CURRENT); nodeMetadata = nodeMetadata.upgradeToCurrentVersion(); ClusterStateLicenseService licenseService = mock(ClusterStateLicenseService.class); BootstrapCheck.BootstrapCheckResult result = new SecurityImplicitBehaviorBootstrapCheck(nodeMetadata, licenseService).check( @@ -110,7 +111,7 @@ public void testUpgradeFrom8xWithImplicitSecuritySettings() throws Exception { public void testUpgradeFrom8xWithExplicitSecuritySettings() throws Exception { final Version previousVersion = VersionUtils.randomVersionBetween(random(), Version.V_8_0_0, null); - NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, Version.CURRENT); + NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(10), previousVersion, IndexVersion.CURRENT); nodeMetadata = nodeMetadata.upgradeToCurrentVersion(); ClusterStateLicenseService licenseService = mock(ClusterStateLicenseService.class); BootstrapCheck.BootstrapCheckResult result = new SecurityImplicitBehaviorBootstrapCheck(nodeMetadata, licenseService).check( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java index c58622afffdd7..31ee57f971b6d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/SecurityTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.engine.InternalEngineFactory; import org.elasticsearch.indices.TestIndexNameExpressionResolver; @@ -188,7 +189,7 @@ protected SSLService getSslService() { private Collection createComponentsUtil(Settings settings) throws Exception { Environment env = TestEnvironment.newEnvironment(settings); - NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(8), Version.CURRENT, Version.CURRENT); + NodeMetadata nodeMetadata = new NodeMetadata(randomAlphaOfLength(8), Version.CURRENT, IndexVersion.CURRENT); ThreadPool threadPool = mock(ThreadPool.class); ClusterService clusterService = mock(ClusterService.class); settings = Security.additionalSettings(settings, true); From ecfd9fd24611c6d156b99a4efb4eee786a281ae2 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Wed, 14 Jun 2023 12:15:58 +0100 Subject: [PATCH 2/5] Fix some tests --- .../org/elasticsearch/env/NodeEnvironment.java | 4 ++-- .../cluster/metadata/MetadataTests.java | 16 ++++++++-------- .../elasticsearch/env/NodeEnvironmentTests.java | 17 +++++++++-------- .../elasticsearch/env/NodeMetadataTests.java | 4 ++++ .../PersistedClusterStateServiceTests.java | 7 ++++--- .../java/org/elasticsearch/test/ESTestCase.java | 13 +++++++++++++ 6 files changed, 40 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 61f05af53e803..92c106c5ac05f 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -524,13 +524,13 @@ static void checkForIndexCompatibility(Logger logger, DataPath... dataPaths) thr if (metadata.oldestIndexVersion().isLegacyIndexVersion()) { throw new IllegalStateException( - "Cannot start this node because it holds metadata for indices created with version [" + "Cannot start this node because it holds metadata for indices with version [" + metadata.oldestIndexVersion() + "] with which this node of version [" + Version.CURRENT + "] is incompatible. Revert this node to version [" + Version.max(Version.CURRENT.minimumCompatibilityVersion(), metadata.previousNodeVersion()) - + "] and delete any indices created in versions earlier than [" + + "] and delete any indices with versions earlier than [" + Version.CURRENT.minimumIndexCompatibilityVersion() + "] before upgrading to version [" + Version.CURRENT diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 462bbad505100..99da52a0d201f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.alias.RandomAliasActionsGenerator; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.plugins.MapperPlugin; @@ -816,28 +817,27 @@ public void testFindMappingsWithFilters() throws IOException { public void testOldestIndexComputation() { Metadata metadata = buildIndicesWithVersions( - new Version[] { Version.V_7_0_0, Version.CURRENT, Version.fromId(Version.CURRENT.id + 1) } + IndexVersion.V_7_0_0, IndexVersion.CURRENT, IndexVersion.fromId(IndexVersion.CURRENT.id() + 1) ).build(); - assertEquals(Version.V_7_0_0, metadata.oldestIndexVersion()); + assertEquals(IndexVersion.V_7_0_0, metadata.oldestIndexVersion()); Metadata.Builder b = Metadata.builder(); - assertEquals(Version.CURRENT, b.build().oldestIndexVersion()); + assertEquals(IndexVersion.CURRENT, b.build().oldestIndexVersion()); Throwable ex = expectThrows( IllegalArgumentException.class, - () -> buildIndicesWithVersions(new Version[] { Version.V_7_0_0, Version.V_EMPTY, Version.fromId(Version.CURRENT.id + 1) }) + () -> buildIndicesWithVersions(IndexVersion.V_7_0_0, IndexVersion.ZERO, IndexVersion.fromId(IndexVersion.CURRENT.id() + 1)) .build() ); assertEquals("[index.version.created] is not present in the index settings for index with UUID [null]", ex.getMessage()); } - private Metadata.Builder buildIndicesWithVersions(Version... indexVersions) { - + private Metadata.Builder buildIndicesWithVersions(IndexVersion... indexVersions) { int lastIndexNum = randomIntBetween(9, 50); Metadata.Builder b = Metadata.builder(); - for (Version indexVersion : indexVersions) { + for (IndexVersion indexVersion : indexVersions) { IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName("index", lastIndexNum)) .settings(settings(indexVersion)) .numberOfShards(1) @@ -852,7 +852,7 @@ private Metadata.Builder buildIndicesWithVersions(Version... indexVersions) { private static IndexMetadata.Builder buildIndexMetadata(String name, String alias, Boolean writeIndex) { return IndexMetadata.builder(name) - .settings(settings(Version.CURRENT)) + .settings(settings(IndexVersion.CURRENT)) .creationDate(randomNonNegativeLong()) .putAlias(AliasMetadata.builder(alias).writeIndex(writeIndex)) .numberOfShards(1) diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index fc9ec28078c34..bb2459ed51f73 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.gateway.PersistedClusterStateService; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -580,7 +581,7 @@ public void testIndexCompatibilityChecks() throws IOException { ); } - Version oldIndexVersion = Version.fromId(between(1, Version.CURRENT.minimumIndexCompatibilityVersion().id - 1)); + IndexVersion oldIndexVersion = IndexVersion.fromId(between(1, IndexVersion.MINIMUM_COMPATIBLE.id() - 1)); Version previousNodeVersion = Version.fromId(between(Version.CURRENT.minimumCompatibilityVersion().id, Version.CURRENT.id - 1)); overrideOldestIndexVersion(oldIndexVersion, previousNodeVersion, env.nodeDataPaths()); @@ -594,17 +595,17 @@ public void testIndexCompatibilityChecks() throws IOException { ex.getMessage(), allOf( containsString("Cannot start this node"), - containsString("it holds metadata for indices created with version [" + oldIndexVersion + "]"), + containsString("it holds metadata for indices with version [" + oldIndexVersion + "]"), containsString("Revert this node to version [" + previousNodeVersion + "]") ) ); // This should work - overrideOldestIndexVersion(Version.CURRENT.minimumIndexCompatibilityVersion(), previousNodeVersion, env.nodeDataPaths()); + overrideOldestIndexVersion(IndexVersion.MINIMUM_COMPATIBLE, previousNodeVersion, env.nodeDataPaths()); checkForIndexCompatibility(logger, env.dataPaths()); // Trying to boot with newer version should pass this check - overrideOldestIndexVersion(NodeMetadataTests.tooNewVersion(), previousNodeVersion, env.nodeDataPaths()); + overrideOldestIndexVersion(NodeMetadataTests.tooNewIndexVersion(), previousNodeVersion, env.nodeDataPaths()); checkForIndexCompatibility(logger, env.dataPaths()); // Simulate empty old index version, attempting to upgrade before 7.17 @@ -714,7 +715,7 @@ public NodeEnvironment newNodeEnvironment(String[] dataPaths, String sharedDataP return new NodeEnvironment(build, TestEnvironment.newEnvironment(build)); } - private static void overrideOldestIndexVersion(Version oldestIndexVersion, Version previousNodeVersion, Path... dataPaths) + private static void overrideOldestIndexVersion(IndexVersion oldestIndexVersion, Version previousNodeVersion, Path... dataPaths) throws IOException { for (final Path dataPath : dataPaths) { final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME); @@ -731,7 +732,7 @@ private static void overrideOldestIndexVersion(Version oldestIndexVersion, Versi ) { final Map commitData = new HashMap<>(userData); commitData.put(NODE_VERSION_KEY, Integer.toString(previousNodeVersion.id)); - commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldestIndexVersion.id)); + commitData.put(OLDEST_INDEX_VERSION_KEY, Integer.toString(oldestIndexVersion.id())); indexWriter.setLiveCommitData(commitData.entrySet()); indexWriter.commit(); } @@ -740,7 +741,7 @@ private static void overrideOldestIndexVersion(Version oldestIndexVersion, Versi } } - private static void removeOldestIndexVersion(Version oldVersion, Path... dataPaths) throws IOException { + private static void removeOldestIndexVersion(IndexVersion oldVersion, Path... dataPaths) throws IOException { for (final Path dataPath : dataPaths) { final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME); if (Files.exists(indexPath)) { @@ -755,7 +756,7 @@ private static void removeOldestIndexVersion(Version oldVersion, Path... dataPat ) ) { final Map commitData = new HashMap<>(userData); - commitData.put(NODE_VERSION_KEY, Integer.toString(oldVersion.id)); + commitData.put(NODE_VERSION_KEY, Integer.toString(oldVersion.id())); commitData.remove(OLDEST_INDEX_VERSION_KEY); indexWriter.setLiveCommitData(commitData.entrySet()); indexWriter.commit(); diff --git a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java index dfaa4e52c0e30..f6fe14fceb925 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeMetadataTests.java @@ -159,6 +159,10 @@ public static Version tooNewVersion() { return Version.fromId(between(Version.CURRENT.id + 1, 99999999)); } + public static IndexVersion tooNewIndexVersion() { + return IndexVersion.fromId(between(IndexVersion.CURRENT.id() + 1, 99999999)); + } + public static Version tooOldVersion() { return Version.fromId(between(1, Version.CURRENT.minimumCompatibilityVersion().id - 1)); } diff --git a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java index 327cafb061c35..5568dd1d211e8 100644 --- a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java @@ -57,6 +57,7 @@ import org.elasticsearch.env.NodeMetadata; import org.elasticsearch.gateway.PersistedClusterStateService.Writer; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.CorruptionUtils; import org.elasticsearch.test.ESTestCase; @@ -1488,12 +1489,12 @@ public void testOldestIndexVersionIsCorrectlySerialized() throws IOException { final Path[] dataPaths2 = createDataPaths(); final Path[] combinedPaths = Stream.concat(Arrays.stream(dataPaths1), Arrays.stream(dataPaths2)).toArray(Path[]::new); - Version oldVersion = Version.fromId(Version.CURRENT.minimumIndexCompatibilityVersion().id - 1); + IndexVersion oldVersion = IndexVersion.fromId(IndexVersion.MINIMUM_COMPATIBLE.id() - 1); - final Version[] indexVersions = new Version[] { oldVersion, Version.CURRENT, Version.fromId(Version.CURRENT.id + 1) }; + final IndexVersion[] indexVersions = new IndexVersion[] { oldVersion, IndexVersion.CURRENT, IndexVersion.fromId(IndexVersion.CURRENT.id() + 1) }; int lastIndexNum = randomIntBetween(9, 50); Metadata.Builder b = Metadata.builder(); - for (Version indexVersion : indexVersions) { + for (IndexVersion indexVersion : indexVersions) { String indexUUID = UUIDs.randomBase64UUID(random()); IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName("index", lastIndexNum)) .putMapping(randomMappingMetadataOrNull()) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 4955750eca8e9..b0d4e0dfd4d52 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -1293,16 +1293,29 @@ public Environment newEnvironment(Settings settings) { } /** Return consistent index settings for the provided index version. */ + @Deprecated public static Settings.Builder settings(Version version) { return Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version); } + /** Return consistent index settings for the provided index version. */ + public static Settings.Builder settings(IndexVersion version) { + return Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, version.id()); + } + /** Return consistent index settings for the provided index version, shard- and replica-count. */ + @Deprecated public static Settings.Builder indexSettings(Version indexVersionCreated, int shards, int replicas) { return settings(indexVersionCreated).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas); } + /** Return consistent index settings for the provided index version, shard- and replica-count. */ + public static Settings.Builder indexSettings(IndexVersion indexVersionCreated, int shards, int replicas) { + return settings(indexVersionCreated).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, shards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicas); + } + /** Return consistent index settings for the provided shard- and replica-count. */ public static Settings.Builder indexSettings(int shards, int replicas) { return Settings.builder() From bb011fb7fd9a30bc0da929c599432bc7335ded7e Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Wed, 14 Jun 2023 13:36:01 +0100 Subject: [PATCH 3/5] Revert "Add JUL bridge (#96683)" This reverts commit 2bdf1bc0d696162636a3a9c426aa3ef2476f8acc. --- gradle/verification-metadata.xml | 5 - .../logging/EvilLoggerConfigurationTests.java | 2 +- server/build.gradle | 2 - ...og4j-LICENSE.txt => log4j-api-LICENSE.txt} | 0 ...{log4j-NOTICE.txt => log4j-api-NOTICE.txt} | 0 server/licenses/log4j-core-LICENSE.txt | 202 ++++++++++++++++++ server/licenses/log4j-core-NOTICE.txt | 20 ++ server/src/main/java/module-info.java | 1 - .../common/logging/LogConfigurator.java | 50 ++--- .../common/logging/LoggingOutputStream.java | 17 +- .../logging/LoggingOutputStreamTests.java | 14 +- 11 files changed, 267 insertions(+), 46 deletions(-) rename server/licenses/{log4j-LICENSE.txt => log4j-api-LICENSE.txt} (100%) rename server/licenses/{log4j-NOTICE.txt => log4j-api-NOTICE.txt} (100%) create mode 100644 server/licenses/log4j-core-LICENSE.txt create mode 100644 server/licenses/log4j-core-NOTICE.txt diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index b673900dc100d..8395ea87e63fb 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2478,11 +2478,6 @@ - - - - - diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java index c0b52c80d89a9..ac033d5091a47 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java @@ -144,7 +144,7 @@ public void testLoggingLevelsFromSettings() throws IOException, UserException { final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); final Configuration config = ctx.getConfiguration(); final Map loggerConfigs = config.getLoggers(); - assertThat(loggerConfigs.size(), equalTo(5)); + assertThat(loggerConfigs.size(), equalTo(3)); assertThat(loggerConfigs, hasKey("")); assertThat(loggerConfigs.get("").getLevel(), equalTo(rootLevel)); assertThat(loggerConfigs, hasKey("foo")); diff --git a/server/build.gradle b/server/build.gradle index 68ed895516fba..3b1885a2fc7ea 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -64,7 +64,6 @@ dependencies { // logging api "org.apache.logging.log4j:log4j-api:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" - implementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" api "net.java.dev.jna:jna:${versions.jna}" @@ -249,7 +248,6 @@ tasks.named("thirdPartyAudit").configure { tasks.named("dependencyLicenses").configure { mapping from: /lucene-.*/, to: 'lucene' - mapping from: /log4j-.*/, to: 'log4j' dependencies = project.configurations.runtimeClasspath.fileCollection { it.group.startsWith('org.elasticsearch') == false || // keep the following org.elasticsearch jars in diff --git a/server/licenses/log4j-LICENSE.txt b/server/licenses/log4j-api-LICENSE.txt similarity index 100% rename from server/licenses/log4j-LICENSE.txt rename to server/licenses/log4j-api-LICENSE.txt diff --git a/server/licenses/log4j-NOTICE.txt b/server/licenses/log4j-api-NOTICE.txt similarity index 100% rename from server/licenses/log4j-NOTICE.txt rename to server/licenses/log4j-api-NOTICE.txt diff --git a/server/licenses/log4j-core-LICENSE.txt b/server/licenses/log4j-core-LICENSE.txt new file mode 100644 index 0000000000000..6279e5206de13 --- /dev/null +++ b/server/licenses/log4j-core-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 1999-2005 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/server/licenses/log4j-core-NOTICE.txt b/server/licenses/log4j-core-NOTICE.txt new file mode 100644 index 0000000000000..bbb5fb3f66e2a --- /dev/null +++ b/server/licenses/log4j-core-NOTICE.txt @@ -0,0 +1,20 @@ +Apache Log4j +Copyright 1999-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +ResolverUtil.java +Copyright 2005-2006 Tim Fennell + +Dumbster SMTP test server +Copyright 2004 Jason Paul Kitchen + +TypeUtil.java +Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams + +picocli (http://picocli.info) +Copyright 2017 Remko Popma + +TimeoutBlockingWaitStrategy.java and parts of Util.java +Copyright 2011 LMAX Ltd. diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index b2b1419b620a8..42125556f771c 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -39,7 +39,6 @@ requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; - requires org.apache.logging.log4j.jul; requires org.apache.lucene.analysis.common; requires org.apache.lucene.backward_codecs; diff --git a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java index 6b7ff4ad35dab..a79c6d1d16940 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java +++ b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.Appender; -import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.config.AbstractConfiguration; @@ -22,13 +21,11 @@ import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; -import org.apache.logging.log4j.core.config.builder.impl.DefaultConfigurationBuilder; import org.apache.logging.log4j.core.config.composite.CompositeConfiguration; import org.apache.logging.log4j.core.config.plugins.util.PluginManager; import org.apache.logging.log4j.core.config.properties.PropertiesConfiguration; import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationBuilder; import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationFactory; -import org.apache.logging.log4j.jul.Log4jBridgeHandler; import org.apache.logging.log4j.status.StatusConsoleListener; import org.apache.logging.log4j.status.StatusData; import org.apache.logging.log4j.status.StatusListener; @@ -225,7 +222,6 @@ public PropertiesConfiguration getConfiguration(final LoggerContext loggerContex properties.setProperty(name, value.replace("%marker", "[%node_name]%marker ")); } } - // end hack return new PropertiesConfigurationBuilder().setConfigurationSource(source) .setRootProperties(properties) @@ -245,8 +241,6 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr }); assert configurations.isEmpty() == false; - configurations.add(createStaticConfiguration(context)); - context.start(new CompositeConfiguration(configurations)); configureLoggerLevels(settings); @@ -263,13 +257,26 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr ); } - Log4jBridgeHandler.install(true, "", true); - // Redirect stdout/stderr to log4j. While we ensure Elasticsearch code does not write to those streams, // third party libraries may do that. Note that we do NOT close the streams because other code may have // grabbed a handle to the streams and intend to write to it, eg log4j for writing to the console - System.setOut(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO), false, StandardCharsets.UTF_8)); - System.setErr(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stderr"), Level.WARN), false, StandardCharsets.UTF_8)); + System.setOut( + new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO, List.of()), false, StandardCharsets.UTF_8) + ); + System.setErr( + new PrintStream( + new LoggingOutputStream( + LogManager.getLogger("stderr"), + Level.WARN, + // MMapDirectory messages come from Lucene, suggesting to users as a warning that they should enable preview features in + // the JDK. Vector logs come from Lucene too, but only if the used explicitly disables the Vector API - no point warning + // in this case. + List.of("MMapDirectory", "VectorUtilProvider", "WARNING: Java vector incubator module is not readable") + ), + false, + StandardCharsets.UTF_8 + ) + ); final Logger rootLogger = LogManager.getRootLogger(); Appender appender = Loggers.findAppender(rootLogger, ConsoleAppender.class); @@ -282,29 +289,6 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr } } - /** - * Creates a log4j configuration that is not changeable by users. - */ - private static AbstractConfiguration createStaticConfiguration(LoggerContext context) { - var builder = new DefaultConfigurationBuilder<>(); - builder.setConfigurationSource(ConfigurationSource.NULL_SOURCE); - builder.setLoggerContext(context); - - // adding filters for confusing Lucene messages - addRegexFilter(builder, "org.apache.lucene.store.MemorySegmentIndexInputProvider", "Using MemorySegmentIndexInput.*"); - addRegexFilter(builder, "org.apache.lucene.util.VectorUtilProvider", ".* incubator module is not readable.*"); - - return builder.build(); - } - - private static void addRegexFilter(DefaultConfigurationBuilder builder, String loggerName, String pattern) { - var filterBuilder = builder.newFilter("RegexFilter", Filter.Result.DENY, Filter.Result.NEUTRAL); - filterBuilder.addAttribute("regex", pattern); - var loggerBuilder = builder.newLogger(loggerName); - loggerBuilder.add(filterBuilder); - builder.add(loggerBuilder); - } - /** * Removes the appender for the console, if one exists. */ diff --git a/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java b/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java index 4092d1d30192d..6fd98259d9250 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java +++ b/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java @@ -15,6 +15,7 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.List; /** * A stream whose output is sent to the configured logger, line by line. @@ -42,9 +43,12 @@ static class Buffer { private final Level level; - LoggingOutputStream(Logger logger, Level level) { + private final List messageFilters; + + LoggingOutputStream(Logger logger, Level level, List messageFilters) { this.logger = logger; this.level = level; + this.messageFilters = messageFilters; } @Override @@ -103,8 +107,17 @@ public void close() { threadLocal = null; } + private void log(String msg) { + for (String filter : messageFilters) { + if (msg.contains(filter)) { + return; + } + } + this.log0(msg); + } + // pkg private for testing - protected void log(String msg) { + protected void log0(String msg) { logger.log(level, msg); } } diff --git a/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java b/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java index 233b1b69cb767..62e3046dc63fd 100644 --- a/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java +++ b/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java @@ -29,20 +29,22 @@ class TestLoggingOutputStream extends LoggingOutputStream { List lines = new ArrayList<>(); TestLoggingOutputStream() { - super(null, null); + super(null, null, messageFilters); } @Override - protected void log(String msg) { + protected void log0(String msg) { lines.add(msg); } } + List messageFilters = new ArrayList<>(); TestLoggingOutputStream loggingStream; PrintStream printStream; @Before public void createStream() { + messageFilters.clear(); loggingStream = new TestLoggingOutputStream(); printStream = new PrintStream(loggingStream, false, StandardCharsets.UTF_8); } @@ -115,4 +117,12 @@ public void testThreadIsolation() throws Exception { printStream.flush(); assertThat(loggingStream.lines, contains("from thread 2", "from thread 1")); } + + public void testMessageFilters() throws Exception { + messageFilters.add("foo bar"); + printStream.println("prefix foo bar suffix"); + printStream.println("non-filtered message"); + printStream.flush(); + assertThat(loggingStream.lines, contains("non-filtered message")); + } } From c9b8a4f46811d6032e02e102e195895127850109 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Wed, 14 Jun 2023 13:51:38 +0100 Subject: [PATCH 4/5] More test fixes --- .../elasticsearch/cluster/metadata/MetadataTests.java | 4 +++- .../java/org/elasticsearch/env/NodeEnvironmentTests.java | 9 +++++---- .../gateway/PersistedClusterStateServiceTests.java | 5 ++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 99da52a0d201f..452d8ea235ee4 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -817,7 +817,9 @@ public void testFindMappingsWithFilters() throws IOException { public void testOldestIndexComputation() { Metadata metadata = buildIndicesWithVersions( - IndexVersion.V_7_0_0, IndexVersion.CURRENT, IndexVersion.fromId(IndexVersion.CURRENT.id() + 1) + IndexVersion.V_7_0_0, + IndexVersion.CURRENT, + IndexVersion.fromId(IndexVersion.CURRENT.id() + 1) ).build(); assertEquals(IndexVersion.V_7_0_0, metadata.oldestIndexVersion()); diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index bb2459ed51f73..14d9090f83fad 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -581,6 +581,7 @@ public void testIndexCompatibilityChecks() throws IOException { ); } + Version oldVersion = Version.fromId(between(1, Version.CURRENT.minimumCompatibilityVersion().id - 1)); IndexVersion oldIndexVersion = IndexVersion.fromId(between(1, IndexVersion.MINIMUM_COMPATIBLE.id() - 1)); Version previousNodeVersion = Version.fromId(between(Version.CURRENT.minimumCompatibilityVersion().id, Version.CURRENT.id - 1)); overrideOldestIndexVersion(oldIndexVersion, previousNodeVersion, env.nodeDataPaths()); @@ -609,7 +610,7 @@ public void testIndexCompatibilityChecks() throws IOException { checkForIndexCompatibility(logger, env.dataPaths()); // Simulate empty old index version, attempting to upgrade before 7.17 - removeOldestIndexVersion(oldIndexVersion, env.nodeDataPaths()); + removeOldestIndexVersion(oldVersion, env.nodeDataPaths()); ex = expectThrows( IllegalStateException.class, @@ -617,7 +618,7 @@ public void testIndexCompatibilityChecks() throws IOException { () -> checkForIndexCompatibility(logger, env.dataPaths()) ); - assertThat(ex.getMessage(), startsWith("cannot upgrade a node from version [" + oldIndexVersion + "] directly")); + assertThat(ex.getMessage(), startsWith("cannot upgrade a node from version [" + oldVersion + "] directly")); assertThat(ex.getMessage(), containsString("upgrade to version [" + Version.CURRENT.minimumCompatibilityVersion())); } } @@ -741,7 +742,7 @@ private static void overrideOldestIndexVersion(IndexVersion oldestIndexVersion, } } - private static void removeOldestIndexVersion(IndexVersion oldVersion, Path... dataPaths) throws IOException { + private static void removeOldestIndexVersion(Version oldVersion, Path... dataPaths) throws IOException { for (final Path dataPath : dataPaths) { final Path indexPath = dataPath.resolve(METADATA_DIRECTORY_NAME); if (Files.exists(indexPath)) { @@ -756,7 +757,7 @@ private static void removeOldestIndexVersion(IndexVersion oldVersion, Path... da ) ) { final Map commitData = new HashMap<>(userData); - commitData.put(NODE_VERSION_KEY, Integer.toString(oldVersion.id())); + commitData.put(NODE_VERSION_KEY, Integer.toString(oldVersion.id)); commitData.remove(OLDEST_INDEX_VERSION_KEY); indexWriter.setLiveCommitData(commitData.entrySet()); indexWriter.commit(); diff --git a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java index 5568dd1d211e8..c1ea3f04170d8 100644 --- a/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/PersistedClusterStateServiceTests.java @@ -1491,7 +1491,10 @@ public void testOldestIndexVersionIsCorrectlySerialized() throws IOException { IndexVersion oldVersion = IndexVersion.fromId(IndexVersion.MINIMUM_COMPATIBLE.id() - 1); - final IndexVersion[] indexVersions = new IndexVersion[] { oldVersion, IndexVersion.CURRENT, IndexVersion.fromId(IndexVersion.CURRENT.id() + 1) }; + final IndexVersion[] indexVersions = new IndexVersion[] { + oldVersion, + IndexVersion.CURRENT, + IndexVersion.fromId(IndexVersion.CURRENT.id() + 1) }; int lastIndexNum = randomIntBetween(9, 50); Metadata.Builder b = Metadata.builder(); for (IndexVersion indexVersion : indexVersions) { From 04127fef9fe427d4b54034301868120a225ad372 Mon Sep 17 00:00:00 2001 From: Simon Cooper Date: Wed, 14 Jun 2023 13:52:01 +0100 Subject: [PATCH 5/5] Revert "Revert "Add JUL bridge (#96683)"" This reverts commit bb011fb7fd9a30bc0da929c599432bc7335ded7e. --- gradle/verification-metadata.xml | 5 + .../logging/EvilLoggerConfigurationTests.java | 2 +- server/build.gradle | 2 + ...og4j-api-LICENSE.txt => log4j-LICENSE.txt} | 0 ...{log4j-api-NOTICE.txt => log4j-NOTICE.txt} | 0 server/licenses/log4j-core-LICENSE.txt | 202 ------------------ server/licenses/log4j-core-NOTICE.txt | 20 -- server/src/main/java/module-info.java | 1 + .../common/logging/LogConfigurator.java | 50 +++-- .../common/logging/LoggingOutputStream.java | 17 +- .../logging/LoggingOutputStreamTests.java | 14 +- 11 files changed, 46 insertions(+), 267 deletions(-) rename server/licenses/{log4j-api-LICENSE.txt => log4j-LICENSE.txt} (100%) rename server/licenses/{log4j-api-NOTICE.txt => log4j-NOTICE.txt} (100%) delete mode 100644 server/licenses/log4j-core-LICENSE.txt delete mode 100644 server/licenses/log4j-core-NOTICE.txt diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 8395ea87e63fb..b673900dc100d 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -2478,6 +2478,11 @@ + + + + + diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java index ac033d5091a47..c0b52c80d89a9 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerConfigurationTests.java @@ -144,7 +144,7 @@ public void testLoggingLevelsFromSettings() throws IOException, UserException { final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); final Configuration config = ctx.getConfiguration(); final Map loggerConfigs = config.getLoggers(); - assertThat(loggerConfigs.size(), equalTo(3)); + assertThat(loggerConfigs.size(), equalTo(5)); assertThat(loggerConfigs, hasKey("")); assertThat(loggerConfigs.get("").getLevel(), equalTo(rootLevel)); assertThat(loggerConfigs, hasKey("foo")); diff --git a/server/build.gradle b/server/build.gradle index 3b1885a2fc7ea..68ed895516fba 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -64,6 +64,7 @@ dependencies { // logging api "org.apache.logging.log4j:log4j-api:${versions.log4j}" api "org.apache.logging.log4j:log4j-core:${versions.log4j}" + implementation "org.apache.logging.log4j:log4j-jul:${versions.log4j}" api "net.java.dev.jna:jna:${versions.jna}" @@ -248,6 +249,7 @@ tasks.named("thirdPartyAudit").configure { tasks.named("dependencyLicenses").configure { mapping from: /lucene-.*/, to: 'lucene' + mapping from: /log4j-.*/, to: 'log4j' dependencies = project.configurations.runtimeClasspath.fileCollection { it.group.startsWith('org.elasticsearch') == false || // keep the following org.elasticsearch jars in diff --git a/server/licenses/log4j-api-LICENSE.txt b/server/licenses/log4j-LICENSE.txt similarity index 100% rename from server/licenses/log4j-api-LICENSE.txt rename to server/licenses/log4j-LICENSE.txt diff --git a/server/licenses/log4j-api-NOTICE.txt b/server/licenses/log4j-NOTICE.txt similarity index 100% rename from server/licenses/log4j-api-NOTICE.txt rename to server/licenses/log4j-NOTICE.txt diff --git a/server/licenses/log4j-core-LICENSE.txt b/server/licenses/log4j-core-LICENSE.txt deleted file mode 100644 index 6279e5206de13..0000000000000 --- a/server/licenses/log4j-core-LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 1999-2005 The Apache Software Foundation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/server/licenses/log4j-core-NOTICE.txt b/server/licenses/log4j-core-NOTICE.txt deleted file mode 100644 index bbb5fb3f66e2a..0000000000000 --- a/server/licenses/log4j-core-NOTICE.txt +++ /dev/null @@ -1,20 +0,0 @@ -Apache Log4j -Copyright 1999-2023 Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -ResolverUtil.java -Copyright 2005-2006 Tim Fennell - -Dumbster SMTP test server -Copyright 2004 Jason Paul Kitchen - -TypeUtil.java -Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams - -picocli (http://picocli.info) -Copyright 2017 Remko Popma - -TimeoutBlockingWaitStrategy.java and parts of Util.java -Copyright 2011 LMAX Ltd. diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 42125556f771c..b2b1419b620a8 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -39,6 +39,7 @@ requires org.apache.logging.log4j; requires org.apache.logging.log4j.core; + requires org.apache.logging.log4j.jul; requires org.apache.lucene.analysis.common; requires org.apache.lucene.backward_codecs; diff --git a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java index a79c6d1d16940..6b7ff4ad35dab 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java +++ b/server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.config.AbstractConfiguration; @@ -21,11 +22,13 @@ import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; +import org.apache.logging.log4j.core.config.builder.impl.DefaultConfigurationBuilder; import org.apache.logging.log4j.core.config.composite.CompositeConfiguration; import org.apache.logging.log4j.core.config.plugins.util.PluginManager; import org.apache.logging.log4j.core.config.properties.PropertiesConfiguration; import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationBuilder; import org.apache.logging.log4j.core.config.properties.PropertiesConfigurationFactory; +import org.apache.logging.log4j.jul.Log4jBridgeHandler; import org.apache.logging.log4j.status.StatusConsoleListener; import org.apache.logging.log4j.status.StatusData; import org.apache.logging.log4j.status.StatusListener; @@ -222,6 +225,7 @@ public PropertiesConfiguration getConfiguration(final LoggerContext loggerContex properties.setProperty(name, value.replace("%marker", "[%node_name]%marker ")); } } + // end hack return new PropertiesConfigurationBuilder().setConfigurationSource(source) .setRootProperties(properties) @@ -241,6 +245,8 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr }); assert configurations.isEmpty() == false; + configurations.add(createStaticConfiguration(context)); + context.start(new CompositeConfiguration(configurations)); configureLoggerLevels(settings); @@ -257,26 +263,13 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr ); } + Log4jBridgeHandler.install(true, "", true); + // Redirect stdout/stderr to log4j. While we ensure Elasticsearch code does not write to those streams, // third party libraries may do that. Note that we do NOT close the streams because other code may have // grabbed a handle to the streams and intend to write to it, eg log4j for writing to the console - System.setOut( - new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO, List.of()), false, StandardCharsets.UTF_8) - ); - System.setErr( - new PrintStream( - new LoggingOutputStream( - LogManager.getLogger("stderr"), - Level.WARN, - // MMapDirectory messages come from Lucene, suggesting to users as a warning that they should enable preview features in - // the JDK. Vector logs come from Lucene too, but only if the used explicitly disables the Vector API - no point warning - // in this case. - List.of("MMapDirectory", "VectorUtilProvider", "WARNING: Java vector incubator module is not readable") - ), - false, - StandardCharsets.UTF_8 - ) - ); + System.setOut(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stdout"), Level.INFO), false, StandardCharsets.UTF_8)); + System.setErr(new PrintStream(new LoggingOutputStream(LogManager.getLogger("stderr"), Level.WARN), false, StandardCharsets.UTF_8)); final Logger rootLogger = LogManager.getRootLogger(); Appender appender = Loggers.findAppender(rootLogger, ConsoleAppender.class); @@ -289,6 +282,29 @@ public FileVisitResult visitFile(final Path file, final BasicFileAttributes attr } } + /** + * Creates a log4j configuration that is not changeable by users. + */ + private static AbstractConfiguration createStaticConfiguration(LoggerContext context) { + var builder = new DefaultConfigurationBuilder<>(); + builder.setConfigurationSource(ConfigurationSource.NULL_SOURCE); + builder.setLoggerContext(context); + + // adding filters for confusing Lucene messages + addRegexFilter(builder, "org.apache.lucene.store.MemorySegmentIndexInputProvider", "Using MemorySegmentIndexInput.*"); + addRegexFilter(builder, "org.apache.lucene.util.VectorUtilProvider", ".* incubator module is not readable.*"); + + return builder.build(); + } + + private static void addRegexFilter(DefaultConfigurationBuilder builder, String loggerName, String pattern) { + var filterBuilder = builder.newFilter("RegexFilter", Filter.Result.DENY, Filter.Result.NEUTRAL); + filterBuilder.addAttribute("regex", pattern); + var loggerBuilder = builder.newLogger(loggerName); + loggerBuilder.add(filterBuilder); + builder.add(loggerBuilder); + } + /** * Removes the appender for the console, if one exists. */ diff --git a/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java b/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java index 6fd98259d9250..4092d1d30192d 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java +++ b/server/src/main/java/org/elasticsearch/common/logging/LoggingOutputStream.java @@ -15,7 +15,6 @@ import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.List; /** * A stream whose output is sent to the configured logger, line by line. @@ -43,12 +42,9 @@ static class Buffer { private final Level level; - private final List messageFilters; - - LoggingOutputStream(Logger logger, Level level, List messageFilters) { + LoggingOutputStream(Logger logger, Level level) { this.logger = logger; this.level = level; - this.messageFilters = messageFilters; } @Override @@ -107,17 +103,8 @@ public void close() { threadLocal = null; } - private void log(String msg) { - for (String filter : messageFilters) { - if (msg.contains(filter)) { - return; - } - } - this.log0(msg); - } - // pkg private for testing - protected void log0(String msg) { + protected void log(String msg) { logger.log(level, msg); } } diff --git a/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java b/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java index 62e3046dc63fd..233b1b69cb767 100644 --- a/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java +++ b/server/src/test/java/org/elasticsearch/common/logging/LoggingOutputStreamTests.java @@ -29,22 +29,20 @@ class TestLoggingOutputStream extends LoggingOutputStream { List lines = new ArrayList<>(); TestLoggingOutputStream() { - super(null, null, messageFilters); + super(null, null); } @Override - protected void log0(String msg) { + protected void log(String msg) { lines.add(msg); } } - List messageFilters = new ArrayList<>(); TestLoggingOutputStream loggingStream; PrintStream printStream; @Before public void createStream() { - messageFilters.clear(); loggingStream = new TestLoggingOutputStream(); printStream = new PrintStream(loggingStream, false, StandardCharsets.UTF_8); } @@ -117,12 +115,4 @@ public void testThreadIsolation() throws Exception { printStream.flush(); assertThat(loggingStream.lines, contains("from thread 2", "from thread 1")); } - - public void testMessageFilters() throws Exception { - messageFilters.add("foo bar"); - printStream.println("prefix foo bar suffix"); - printStream.println("non-filtered message"); - printStream.flush(); - assertThat(loggingStream.lines, contains("non-filtered message")); - } }