Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement ml.estimate_model_memory.json #4530

Merged
merged 4 commits into from
Apr 16, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion src/ApiGenerator/Configuration/CodeConfiguration.cs
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ public static class CodeConfiguration
"cluster.delete_component_template.json",
"cluster.get_component_template.json",
"cluster.put_component_template.json",
"ml.estimate_model_memory.json",
"ml.set_upgrade_mode.json",
"security.get_builtin_privileges.json",
"transform.delete_transform.json",
Expand Down
8 changes: 8 additions & 0 deletions src/Nest/Descriptors.MachineLearning.cs
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,14 @@ protected DeleteModelSnapshotDescriptor(): base()
// Request parameters
}

///<summary>Descriptor for EstimateModelMemory</summary>
public partial class EstimateModelMemoryDescriptor<TDocument> : RequestDescriptorBase<EstimateModelMemoryDescriptor<TDocument>, EstimateModelMemoryRequestParameters, IEstimateModelMemoryRequest>, IEstimateModelMemoryRequest
{
internal override ApiUrls ApiUrls => ApiUrlsLookups.MachineLearningEstimateModelMemory;
// values part of the url path
// Request parameters
}

///<summary>Descriptor for FlushJob <para>https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-flush-job.html</para></summary>
public partial class FlushJobDescriptor : RequestDescriptorBase<FlushJobDescriptor, FlushJobRequestParameters, IFlushJobRequest>, IFlushJobRequest
{
Expand Down
26 changes: 26 additions & 0 deletions src/Nest/ElasticClient.MachineLearning.cs
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,32 @@ internal MachineLearningNamespace(ElasticClient client): base(client)
/// </summary>
public Task<DeleteModelSnapshotResponse> DeleteModelSnapshotAsync(IDeleteModelSnapshotRequest request, CancellationToken ct = default) => DoRequestAsync<IDeleteModelSnapshotRequest, DeleteModelSnapshotResponse>(request, request.RequestParameters, ct);
/// <summary>
/// <c>POST</c> request to the <c>ml.estimate_model_memory</c> API, read more about this API online:
/// <para></para>
/// <a></a>
/// </summary>
public EstimateModelMemoryResponse EstimateModelMemory<TDocument>(Func<EstimateModelMemoryDescriptor<TDocument>, IEstimateModelMemoryRequest> selector)
where TDocument : class => EstimateModelMemory(selector.InvokeOrDefault(new EstimateModelMemoryDescriptor<TDocument>()));
/// <summary>
/// <c>POST</c> request to the <c>ml.estimate_model_memory</c> API, read more about this API online:
/// <para></para>
/// <a></a>
/// </summary>
public Task<EstimateModelMemoryResponse> EstimateModelMemoryAsync<TDocument>(Func<EstimateModelMemoryDescriptor<TDocument>, IEstimateModelMemoryRequest> selector, CancellationToken ct = default)
where TDocument : class => EstimateModelMemoryAsync(selector.InvokeOrDefault(new EstimateModelMemoryDescriptor<TDocument>()), ct);
/// <summary>
/// <c>POST</c> request to the <c>ml.estimate_model_memory</c> API, read more about this API online:
/// <para></para>
/// <a></a>
/// </summary>
public EstimateModelMemoryResponse EstimateModelMemory(IEstimateModelMemoryRequest request) => DoRequest<IEstimateModelMemoryRequest, EstimateModelMemoryResponse>(request, request.RequestParameters);
/// <summary>
/// <c>POST</c> request to the <c>ml.estimate_model_memory</c> API, read more about this API online:
/// <para></para>
/// <a></a>
/// </summary>
public Task<EstimateModelMemoryResponse> EstimateModelMemoryAsync(IEstimateModelMemoryRequest request, CancellationToken ct = default) => DoRequestAsync<IEstimateModelMemoryRequest, EstimateModelMemoryResponse>(request, request.RequestParameters, ct);
/// <summary>
/// <c>POST</c> request to the <c>ml.flush_job</c> API, read more about this API online:
/// <para></para>
/// <a href = "https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-flush-job.html">https://www.elastic.co/guide/en/elasticsearch/reference/current/ml-flush-job.html</a>
Expand Down
14 changes: 14 additions & 0 deletions src/Nest/Requests.MachineLearning.cs
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,20 @@ protected DeleteModelSnapshotRequest(): base()
// Request parameters
}

[InterfaceDataContract]
public partial interface IEstimateModelMemoryRequest : IRequest<EstimateModelMemoryRequestParameters>
{
}

///<summary>Request for EstimateModelMemory</summary>
public partial class EstimateModelMemoryRequest : PlainRequestBase<EstimateModelMemoryRequestParameters>, IEstimateModelMemoryRequest
{
protected IEstimateModelMemoryRequest Self => this;
internal override ApiUrls ApiUrls => ApiUrlsLookups.MachineLearningEstimateModelMemory;
// values part of the url path
// Request parameters
}

[InterfaceDataContract]
public partial interface IFlushJobRequest : IRequest<FlushJobRequestParameters>
{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Runtime.Serialization;
using Elasticsearch.Net.Utf8Json;

namespace Nest
{
[MapsApi("ml.estimate_model_memory.json")]
[ReadAs(typeof(EstimateModelMemoryRequest))]
public partial interface IEstimateModelMemoryRequest
{
/// <summary>
/// For a list of the properties that you can specify in the analysis_config component of the body
/// of this API, see analysis_config.
/// </summary>
[DataMember(Name ="analysis_config")]
IAnalysisConfig AnalysisConfig { get; set; }

/// <summary>
/// Estimates of the cardinality that will be observed for fields over the whole time period that
/// the job analyzes data. To produce a good answer, values must be provided for fields referenced
/// in the by_field_name, over_field_name and partition_field_name of any detectors. It does not matter
/// if values are provided for other fields. If no detectors have a by_field_name, over_field_name or
/// partition_field_name then overall_cardinality can be omitted from the request.
/// </summary>
[DataMember(Name = "overall_cardinality")]
IOverallCardinality OverallCardinality { get; set; }

/// <summary>
/// Estimates of the highest cardinality in a single bucket that will be observed for influencer
/// fields over the time period that the job analyzes data. To produce a good answer, values must
/// be provided for all influencer fields. It does not matter if values are provided for fields
/// that are not listed as influencers. If there are no influencers then max_bucket_cardinality
/// can be omitted from the request.
/// </summary>
[DataMember(Name = "max_bucket_cardinality")]
IMaxBucketCardinality MaxBucketCardinality { get; set; }
}

public partial class EstimateModelMemoryRequest
{
/// <inheritdoc />
public IAnalysisConfig AnalysisConfig { get; set; }

/// <inheritdoc />
public IOverallCardinality OverallCardinality { get; set; }

/// <inheritdoc />
public IMaxBucketCardinality MaxBucketCardinality { get; set; }
}

public partial class EstimateModelMemoryDescriptor<TDocument> where TDocument : class
{
IAnalysisConfig IEstimateModelMemoryRequest.AnalysisConfig { get; set; }
IOverallCardinality IEstimateModelMemoryRequest.OverallCardinality { get; set; }
IMaxBucketCardinality IEstimateModelMemoryRequest.MaxBucketCardinality { get; set; }

/// <inheritdoc />
public EstimateModelMemoryDescriptor<TDocument> AnalysisConfig(Func<AnalysisConfigDescriptor<TDocument>, IAnalysisConfig> selector) =>
Assign(selector, (a, v) => a.AnalysisConfig = v?.Invoke(new AnalysisConfigDescriptor<TDocument>()));

/// <inheritdoc />
public EstimateModelMemoryDescriptor<TDocument> OverallCardinality(Func<OverallCardinalityDescriptor<TDocument>, IPromise<IOverallCardinality>> analyzerSelector) =>
Assign(analyzerSelector, (a, v) => a.OverallCardinality = v?.Invoke(new OverallCardinalityDescriptor<TDocument>())?.Value);

/// <inheritdoc />
public EstimateModelMemoryDescriptor<TDocument> MaxBucketCardinality(Func<MaxBucketCardinalityDescriptor<TDocument>, IPromise<IMaxBucketCardinality>> analyzerSelector) =>
Assign(analyzerSelector, (a, v) => a.MaxBucketCardinality = v?.Invoke(new MaxBucketCardinalityDescriptor<TDocument>())?.Value);
}

[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<OverallCardinality, IOverallCardinality, Field, long>))]
public interface IOverallCardinality : IIsADictionary<Field, long> { }

public class OverallCardinality : IsADictionaryBase<Field, long>, IOverallCardinality
{
public OverallCardinality() { }

public OverallCardinality(IDictionary<Field, long> container) : base(container) { }

public void Add(Field field, long cardinality) => BackingDictionary.Add(field, cardinality);
}

public class OverallCardinality<T> : OverallCardinality where T : class
{
public void Add<TValue>(Expression<Func<T, TValue>> field, long cardinality) => BackingDictionary.Add(field, cardinality);
}

public class OverallCardinalityDescriptor<T> : IsADictionaryDescriptorBase<OverallCardinalityDescriptor<T>, IOverallCardinality, Field, long> where T : class
{
public OverallCardinalityDescriptor() : base(new OverallCardinality()) { }

public OverallCardinalityDescriptor<T> Field(Field field, long cardinality) => Assign(field, cardinality);

public OverallCardinalityDescriptor<T> Field<TValue>(Expression<Func<T, TValue>> field, long cardinality) => Assign(field, cardinality);
}

[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<MaxBucketCardinality, IMaxBucketCardinality, Field, long>))]
public interface IMaxBucketCardinality : IIsADictionary<Field, long> { }

public class MaxBucketCardinality : IsADictionaryBase<Field, long>, IMaxBucketCardinality
{
public MaxBucketCardinality() { }

public MaxBucketCardinality(IDictionary<Field, long> container) : base(container) { }

public MaxBucketCardinality(Dictionary<Field, long> container) : base(container) { }
codebrain marked this conversation as resolved.
Show resolved Hide resolved

public void Add(Field field, long cardinality) => BackingDictionary.Add(field, cardinality);
}

public class MaxBucketCardinality<T> : MaxBucketCardinality where T : class
{
public void Add<TValue>(Expression<Func<T, TValue>> field, long cardinality) => BackingDictionary.Add(field, cardinality);
}

public class MaxBucketCardinalityDescriptor<T> : IsADictionaryDescriptorBase<MaxBucketCardinalityDescriptor<T>, IMaxBucketCardinality, Field, long> where T : class
{
public MaxBucketCardinalityDescriptor() : base(new MaxBucketCardinality()) { }

public MaxBucketCardinalityDescriptor<T> Field(Field field, long cardinality) => Assign(field, cardinality);

public MaxBucketCardinalityDescriptor<T> Field<TValue>(Expression<Func<T, TValue>> field, long cardinality) => Assign(field, cardinality);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
using System.Runtime.Serialization;

namespace Nest
{
public class EstimateModelMemoryResponse : ResponseBase
{
[DataMember(Name ="model_memory_estimate")]
public string ModelMemoryEstimate { get; internal set; }
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
using System;
using Elastic.Xunit.XunitPlumbing;
using Elasticsearch.Net;
using Nest;
using Tests.Core.Extensions;
using Tests.Domain;
using Tests.Framework.EndpointTests.TestState;
using static Nest.Infer;

namespace Tests.XPack.MachineLearning.EstimateModelMemory
{
[SkipVersion("<7.7.0", "Introduced in 7.7.0")]
public class EstimateModelMemoryApiTests : MachineLearningIntegrationTestBase<EstimateModelMemoryResponse, IEstimateModelMemoryRequest, EstimateModelMemoryDescriptor<Metric>, EstimateModelMemoryRequest>
{
public EstimateModelMemoryApiTests(MachineLearningCluster cluster, EndpointUsage usage) : base(cluster, usage) { }

protected override bool ExpectIsValid => true;
protected override int ExpectStatusCode => 200;
protected override Func<EstimateModelMemoryDescriptor<Metric>, IEstimateModelMemoryRequest> Fluent => f => f
.AnalysisConfig(a => a
.BucketSpan("30m")
.Latency("0s")
.Detectors(d => d.Sum(c => c.FieldName(r => r.Total)))
)
.OverallCardinality(m =>
m.Field(f => f.Response, 50)
.Field(f => f.Accept, 10)
)
.MaxBucketCardinality(m =>
m.Field(f => f.Response, 500)
.Field(f => f.Accept, 100)
);

protected override HttpMethod HttpMethod => HttpMethod.POST;

protected override object ExpectJson => new
{
analysis_config = new
{
bucket_span = "30m",
detectors = new[]
{
new
{
function = "sum",
field_name = "total"
}
},
latency = "0s",
},
overall_cardinality = new
{
response = 50,
accept = 10,
},
max_bucket_cardinality = new
{
response = 500,
accept = 100,
}
};

protected override EstimateModelMemoryRequest Initializer => new EstimateModelMemoryRequest
{
AnalysisConfig = new AnalysisConfig
{
BucketSpan = "30m",
Latency = "0s",
Detectors = new[]
{
new SumDetector
{
FieldName = Field<Metric>(f => f.Total)
}
}
},
OverallCardinality = new OverallCardinality
{
{ Field<Metric>(f => f.Response), 50 },
{ Field<Metric>(f => f.Accept), 10 }
},
MaxBucketCardinality = new MaxBucketCardinality
{
{ Field<Metric>(f => f.Response), 500 },
{ Field<Metric>(f => f.Accept), 100 }
}
};

protected override string UrlPath => $"/_ml/anomaly_detectors/_estimate_model_memory";

protected override LazyResponses ClientUsage() => Calls(
(client, f) => client.MachineLearning.EstimateModelMemory(f),
(client, f) => client.MachineLearning.EstimateModelMemoryAsync(f),
(client, r) => client.MachineLearning.EstimateModelMemory(r),
(client, r) => client.MachineLearning.EstimateModelMemoryAsync(r)
);

protected override EstimateModelMemoryDescriptor<Metric> NewDescriptor() => new EstimateModelMemoryDescriptor<Metric>();

protected override void ExpectResponse(EstimateModelMemoryResponse response) => response.ShouldBeValid();
}
codebrain marked this conversation as resolved.
Show resolved Hide resolved
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
using System.Threading.Tasks;
using Elastic.Xunit.XunitPlumbing;
using Nest;
using Tests.Framework.EndpointTests;
using static Tests.Framework.EndpointTests.UrlTester;

namespace Tests.XPack.MachineLearning.EstimateModelMemory
{
public class EstimateModelMemoryUrlTests : UrlTestsBase
{
[U] public override async Task Urls() => await POST("/_ml/anomaly_detectors/_estimate_model_memory")
.Fluent(c => c.MachineLearning.EstimateModelMemory(new EstimateModelMemoryRequest()))
.Request(c => c.MachineLearning.EstimateModelMemory(new EstimateModelMemoryRequest()))
.FluentAsync(c => c.MachineLearning.EstimateModelMemoryAsync(new EstimateModelMemoryRequest()))
.RequestAsync(c => c.MachineLearning.EstimateModelMemoryAsync(new EstimateModelMemoryRequest()));
}
}