Add license feature usage api (#59342) (#59571)

This commit adds a new api to track when gold+ features are used within
x-pack. The tracking is done internally whenever a feature is checked
against the current license. The output of the api is a list of each
used feature, which includes the name, license level, and last time it
was used. In addition to a unit test for the tracking, a rest test is
added which ensures starting up a default configured node does not
result in any features registering as used.

There are a couple features which currently do not work well with the
tracking, as they are checked in a manner that makes them look always
used. Those features will be fixed in followups, and in this PR they are
omitted from the feature usage output.

libs/core/src/main/java/org/elasticsearch/common/MemoizedSupplier.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common;
+
+import java.util.function.Supplier;
+
+public class MemoizedSupplier<T> implements Supplier<T> {
+    private Supplier<T> supplier;
+    private T value;
+
+    public MemoizedSupplier(Supplier<T> supplier) {
+        this.supplier = supplier;
+    }
+
+    @Override
+    public T get() {
+        if (supplier != null) {
+            value = supplier.get();
+            supplier = null;
+        }
+        return value;
+    }
+}

x-pack/plugin/core/build.gradle

@@ -7,6 +7,7 @@ import java.nio.file.Paths
 apply plugin: 'elasticsearch.esplugin'
 apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.internal-cluster-test'
+apply plugin: 'elasticsearch.yaml-rest-test'
 
 archivesBaseName = 'x-pack-core'
 
@@ -57,6 +58,8 @@ dependencies {
     transitive = false
   }
 
+  yamlRestTestImplementation project(':x-pack:plugin:core')
+
 }
 
 ext.expansions = [
@@ -143,7 +146,18 @@ thirdPartyAudit.ignoreMissingClasses(
   'javax.servlet.ServletContextListener'
 )
 
-// xpack modules are installed in real clusters as the meta plugin, so
-// installing them as individual plugins for integ tests doesn't make sense,
-// so we disable integ tests
-integTest.enabled = false
+restResources {
+  restApi {
+    includeCore '*'
+  }
+}
+
+testClusters.yamlRestTest {
+  testDistribution = 'default'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.license.self_generated.type', 'trial'
+  keystore 'bootstrap.password', 'x-pack-test-password'
+  user username: "x_pack_rest_user", password: "x-pack-test-password"
+}
+
+testingConventions.enabled = false

x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetFeatureUsageRequest.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.license;
+
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.common.io.stream.StreamInput;
+
+import java.io.IOException;
+
+public class GetFeatureUsageRequest extends ActionRequest {
+
+    public GetFeatureUsageRequest() {}
+
+    public GetFeatureUsageRequest(StreamInput in) throws IOException {
+        super(in);
+    }
+
+    @Override
+    public ActionRequestValidationException validate() {
+        return null;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/license/GetFeatureUsageResponse.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.license;
+
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ToXContentObject;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.Collections;
+import java.util.List;
+
+public class GetFeatureUsageResponse extends ActionResponse implements ToXContentObject {
+
+    public static class FeatureUsageInfo implements Writeable {
+        public final String name;
+        public final ZonedDateTime lastUsedTime;
+        public final String licenseLevel;
+
+        public FeatureUsageInfo(String name, ZonedDateTime lastUsedTime, String licenseLevel) {
+            this.name = name;
+            this.lastUsedTime = lastUsedTime;
+            this.licenseLevel = licenseLevel;
+        }
+
+        public FeatureUsageInfo(StreamInput in) throws IOException {
+            this.name = in.readString();
+            this.lastUsedTime = ZonedDateTime.ofInstant(Instant.ofEpochSecond(in.readLong()), ZoneOffset.UTC);
+            this.licenseLevel = in.readString();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            out.writeString(name);
+            out.writeLong(lastUsedTime.toEpochSecond());
+            out.writeString(licenseLevel);
+        }
+    }
+
+    private List<FeatureUsageInfo> features;
+
+    public GetFeatureUsageResponse(List<FeatureUsageInfo> features) {
+        this.features = Collections.unmodifiableList(features);
+    }
+
+    public GetFeatureUsageResponse(StreamInput in) throws IOException {
+        this.features = in.readList(FeatureUsageInfo::new);
+    }
+
+    public List<FeatureUsageInfo> getFeatures() {
+        return features;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeList(features);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.startArray("features");
+        for (FeatureUsageInfo feature : features) {
+            builder.startObject();
+            builder.field("name", feature.name);
+            builder.field("last_used", feature.lastUsedTime.toString());
+            builder.field("license_level", feature.licenseLevel);
+            builder.endObject();
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/license/Licensing.java

@@ -66,7 +66,8 @@ public Licensing(Settings settings) {
                 new ActionHandler<>(PostStartTrialAction.INSTANCE, TransportPostStartTrialAction.class),
                 new ActionHandler<>(GetTrialStatusAction.INSTANCE, TransportGetTrialStatusAction.class),
                 new ActionHandler<>(PostStartBasicAction.INSTANCE, TransportPostStartBasicAction.class),
-                new ActionHandler<>(GetBasicStatusAction.INSTANCE, TransportGetBasicStatusAction.class));
+                new ActionHandler<>(GetBasicStatusAction.INSTANCE, TransportGetBasicStatusAction.class),
+                new ActionHandler<>(TransportGetFeatureUsageAction.TYPE, TransportGetFeatureUsageAction.class));
     }
 
     @Override
@@ -81,6 +82,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
         handlers.add(new RestGetBasicStatus());
         handlers.add(new RestPostStartTrialLicense());
         handlers.add(new RestPostStartBasicLicense());
+        handlers.add(new RestGetFeatureUsageAction());
         return handlers;
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/license/RestGetFeatureUsageAction.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.license;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.action.RestToXContentListener;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.elasticsearch.rest.RestRequest.Method.GET;
+
+public class RestGetFeatureUsageAction extends BaseRestHandler {
+
+    @Override
+    public String getName() {
+        return "get_feature_usage";
+    }
+
+    @Override
+    public List<Route> routes() {
+        return Collections.singletonList(new Route(GET, "/_license/feature_usage"));
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        return channel -> client.execute(TransportGetFeatureUsageAction.TYPE, new GetFeatureUsageRequest(),
+            new RestToXContentListener<>(channel));
+    }
+}

x-pack/plugin/core/src/main/java/org/elasticsearch/license/TransportGetFeatureUsageAction.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.license;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.action.support.HandledTransportAction;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.transport.TransportService;
+
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+public class TransportGetFeatureUsageAction extends HandledTransportAction<GetFeatureUsageRequest, GetFeatureUsageResponse> {
+
+    public static final ActionType<GetFeatureUsageResponse> TYPE =
+        new ActionType<>("cluster:admin/xpack/license/feature_usage", GetFeatureUsageResponse::new);
+
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public TransportGetFeatureUsageAction(TransportService transportService, ActionFilters actionFilters,
+                                          XPackLicenseState licenseState) {
+        super(TYPE.name(), transportService, actionFilters, GetFeatureUsageRequest::new);
+        this.licenseState = licenseState;
+    }
+
+
+    @Override
+    protected void doExecute(Task task, GetFeatureUsageRequest request, ActionListener<GetFeatureUsageResponse> listener) {
+        Map<XPackLicenseState.Feature, Long> featureUsage = licenseState.getLastUsed();
+        List<GetFeatureUsageResponse.FeatureUsageInfo> usageInfos = new ArrayList<>();
+        for (Map.Entry<XPackLicenseState.Feature, Long> entry : featureUsage.entrySet()) {
+            XPackLicenseState.Feature feature = entry.getKey();
+            String name = feature.name().toLowerCase(Locale.ROOT);
+            ZonedDateTime lastUsedTime = Instant.ofEpochMilli(entry.getValue()).atZone(ZoneOffset.UTC);
+            String licenseLevel = feature.minimumOperationMode.name().toLowerCase(Locale.ROOT);
+            usageInfos.add(new GetFeatureUsageResponse.FeatureUsageInfo(name, lastUsedTime, licenseLevel));
+        }
+        listener.onResponse(new GetFeatureUsageResponse(usageInfos));
+    }
+}