Add PerFieldStoredFieldsFormat to allow multiple stored field formats

docs/changelog/138299.yaml

@@ -0,0 +1,5 @@
+pr: 138299
+summary: Add `PerFieldStoredFieldsFormat` to allow multiple stored field formats
+area: Codec
+type: enhancement
+issues: []

server/src/main/java/module-info.java

@@ -245,6 +245,7 @@
     exports org.elasticsearch.index.codec;
     exports org.elasticsearch.index.codec.tsdb;
     exports org.elasticsearch.index.codec.bloomfilter;
+    exports org.elasticsearch.index.codec.storedfields;
     exports org.elasticsearch.index.codec.zstd;
     exports org.elasticsearch.index.engine;
     exports org.elasticsearch.index.fielddata;
@@ -468,6 +469,13 @@
             org.elasticsearch.index.codec.vectors.es93.ES93BinaryQuantizedVectorsFormat,
             org.elasticsearch.index.codec.vectors.es93.ES93HnswVectorsFormat,
             org.elasticsearch.index.codec.vectors.es93.ES93HnswBinaryQuantizedVectorsFormat;
+    provides org.elasticsearch.index.codec.storedfields.ESStoredFieldsFormat
+        with
+            org.elasticsearch.index.codec.storedfields.ESZstd814StoredFieldsFormat,
+            org.elasticsearch.index.codec.storedfields.ESLucene90StoredFieldsFormat,
+            org.elasticsearch.index.codec.bloomfilter.ES93BloomFilterStoredFieldsFormat;
+
+    uses org.elasticsearch.index.codec.storedfields.ESStoredFieldsFormat;
 
     provides org.apache.lucene.codecs.Codec
         with

server/src/main/java/org/elasticsearch/index/codec/storedfields/ESLucene90StoredFieldsFormat.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.codec.storedfields;
+
+import org.apache.lucene.codecs.lucene103.Lucene103Codec;
+import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter;
+
+import java.util.Set;
+
+/**
+ * Simple wrapper for Lucene90StoredFieldsFormat that allows it to be loaded through SPI
+ */
+public class ESLucene90StoredFieldsFormat extends FilterESStoredFieldsFormat {
+    public static final Set<String> FILE_EXTENSIONS = Set.of(
+        Lucene90CompressingStoredFieldsWriter.FIELDS_EXTENSION,
+        Lucene90CompressingStoredFieldsWriter.INDEX_EXTENSION,
+        Lucene90CompressingStoredFieldsWriter.META_EXTENSION
+    );
+
+    public ESLucene90StoredFieldsFormat() {
+        this(Lucene103Codec.Mode.BEST_SPEED);
+    }
+
+    public ESLucene90StoredFieldsFormat(Lucene103Codec.Mode mode) {
+        super(
+            "ESLucene90StoredFieldsFormat",
+            new Lucene90StoredFieldsFormat(
+                mode == Lucene103Codec.Mode.BEST_COMPRESSION
+                    ? Lucene90StoredFieldsFormat.Mode.BEST_COMPRESSION
+                    : Lucene90StoredFieldsFormat.Mode.BEST_SPEED
+            )
+        );
+    }
+
+    @Override
+    protected Set<String> getFileExtensions() {
+        return FILE_EXTENSIONS;
+    }
+}

server/src/main/java/org/elasticsearch/index/codec/storedfields/ESStoredFieldsFormat.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.codec.storedfields;
+
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.util.NamedSPILoader;
+
+import java.util.Set;
+
+/**
+ * A {@link StoredFieldsFormat} that can be loaded via SPI and provides a name for identification.
+ * This is required because {@link PerFieldStoredFieldsFormat} uses SPI to load stored field formats
+ * when reading fields.
+ */
+public abstract class ESStoredFieldsFormat extends StoredFieldsFormat implements NamedSPILoader.NamedSPI {
+    private static final class Holder {
+        public static final NamedSPILoader<ESStoredFieldsFormat> LOADER = new NamedSPILoader<>(ESStoredFieldsFormat.class);
+
+        private Holder() {}
+
+        static NamedSPILoader<ESStoredFieldsFormat> getLoader() {
+            if (LOADER == null) {
+                throw new IllegalStateException(
+                    "You tried to lookup a ESStoredFieldsFormat by name before all formats could be initialized."
+                );
+            }
+            return LOADER;
+        }
+    }
+
+    public static ESStoredFieldsFormat forName(String name) {
+        return Holder.getLoader().lookup(name);
+    }
+
+    /**
+     * Unique name that's used to retrieve this format when reading the index.
+     */
+    private final String name;
+
+    protected ESStoredFieldsFormat(String name) {
+        NamedSPILoader.checkServiceName(name);
+        this.name = name;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Returns the set of file fileExtensions that this stored fields format would write to disk.
+     */
+    protected abstract Set<String> getFileExtensions();
+}

server/src/main/java/org/elasticsearch/index/codec/storedfields/ESZstd814StoredFieldsFormat.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.codec.storedfields;
+
+import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsWriter;
+import org.elasticsearch.index.codec.zstd.Zstd814StoredFieldsFormat;
+
+import java.util.Set;
+
+/**
+ * Simple wrapper for Lucene90StoredFieldsFormat that uses zstd for compression. Allowing to be loaded through SPI.
+ */
+public class ESZstd814StoredFieldsFormat extends FilterESStoredFieldsFormat {
+    public static final Set<String> FILE_EXTENSIONS = Set.of(
+        Lucene90CompressingStoredFieldsWriter.FIELDS_EXTENSION,
+        Lucene90CompressingStoredFieldsWriter.INDEX_EXTENSION,
+        Lucene90CompressingStoredFieldsWriter.META_EXTENSION
+    );
+
+    public ESZstd814StoredFieldsFormat() {
+        this(Zstd814StoredFieldsFormat.Mode.BEST_SPEED);
+    }
+
+    public ESZstd814StoredFieldsFormat(Zstd814StoredFieldsFormat.Mode mode) {
+        super("ESZstd814StoredFieldsFormat", mode.getFormat());
+    }
+
+    @Override
+    protected Set<String> getFileExtensions() {
+        return FILE_EXTENSIONS;
+    }
+}

server/src/main/java/org/elasticsearch/index/codec/storedfields/FilterESStoredFieldsFormat.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.codec.storedfields;
+
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.StoredFieldsReader;
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentInfo;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+
+import java.io.IOException;
+
+abstract class FilterESStoredFieldsFormat extends ESStoredFieldsFormat {
+    private final StoredFieldsFormat delegate;
+
+    FilterESStoredFieldsFormat(String name, StoredFieldsFormat delegate) {
+        super(name);
+        this.delegate = delegate;
+    }
+
+    @Override
+    public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
+        return delegate.fieldsReader(directory, si, fn, context);
+    }
+
+    @Override
+    public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
+        return delegate.fieldsWriter(directory, si, context);
+    }
+}