Set null value handling per request

[chrisdrobison]

I know that null value handling is kind of an all or nothing thing with JSON.NET, but I've come across an interesting issue. For things like update and index I want to include null values, as they pertain to fields in documents I'm interested in. In my case, I'm passing in both POCOs and dictionaries depending on what I want to do and I want the null values to be included in both. I know that for POCOs there is JsonPropertyAttribute to fix this. I not aware of something similar dictionaries. For searching, the null value stuff gets in the way. For example, with I have null values included, the library generates this for one search I'm doing:

{
    "timeout": null,
    "from": 0,
    "size": 200,
    "explain": null,
    "version": null,
    "track_scores": null,
    "min_score": null,
    "indices_boost": null,
    "sort": [{
        "date_created": {
            "ignore_unmapped": null,
            "missing": null,
            "order": "desc",
            "mode": null,
            "nested_filter": null,
            "nested_path": null
        }
    }],
    "facets": null,
    "suggest": null,
    "highlight": null,
    "rescore": null,
    "fields": null,
    "script_fields": null,
    "_source": null,
    "aggs": null,
    "query": {
        "bool": null,
        "match_all": null,
        "term": null,
        "wildcard": null,
        "prefix": null,
        "boosting": null,
        "ids": null,
        "custom_score": null,
        "custom_filters_score": null,
        "custom_boost_factor": null,
        "constant_score": null,
        "dis_max": null,
        "filtered": {
            "query": {
                "bool": null,
                "match_all": {
                    "boost": null,
                    "norm_field": null
                },
                "term": null,
                "wildcard": null,
                "prefix": null,
                "boosting": null,
                "ids": null,
                "custom_score": null,
                "custom_filters_score": null,
                "custom_boost_factor": null,
                "constant_score": null,
                "dis_max": null,
                "filtered": null,
                "multi_match": null,
                "match": null,
                "fuzzy": null,
                "geo_shape": null,
                "common": null,
                "terms": null,
                "range": null,
                "regexp": null,
                "has_child": null,
                "has_parent": null,
                "span_term": null,
                "flt": null,
                "simple_query_string": null,
                "query_string": null,
                "mlt": null,
                "span_first": null,
                "span_or": null,
                "span_near": null,
                "span_not": null,
                "span_multi": null,
                "top_children": null,
                "nested": null,
                "indices": null,
                "function_score": null,
                "template": null
            },
            "filter": {
                "bool": null,
                "exists": null,
                "missing": null,
                "ids": null,
                "geo_bounding_box": null,
                "geo_distance": null,
                "geohash_cell": null,
                "geo_distance_range": null,
                "geo_polygon": null,
                "geo_shape": null,
                "limit": null,
                "type": null,
                "match_all": null,
                "has_child": null,
                "has_parent": null,
                "range": null,
                "prefix": null,
                "term": null,
                "terms": null,
                "fquery": {
                    "query": {
                        "bool": {
                            "must": [{
                                "bool": null,
                                "match_all": null,
                                "term": {
                                    "instanceid": {
                                        "value": "CDR",
                                        "boost": null
                                    }
                                },
                                "wildcard": null,
                                "prefix": null,
                                "boosting": null,
                                "ids": null,
                                "custom_score": null,
                                "custom_filters_score": null,
                                "custom_boost_factor": null,
                                "constant_score": null,
                                "dis_max": null,
                                "filtered": null,
                                "multi_match": null,
                                "match": null,
                                "fuzzy": null,
                                "geo_shape": null,
                                "common": null,
                                "terms": null,
                                "range": null,
                                "regexp": null,
                                "has_child": null,
                                "has_parent": null,
                                "span_term": null,
                                "flt": null,
                                "simple_query_string": null,
                                "query_string": null,
                                "mlt": null,
                                "span_first": null,
                                "span_or": null,
                                "span_near": null,
                                "span_not": null,
                                "span_multi": null,
                                "top_children": null,
                                "nested": null,
                                "indices": null,
                                "function_score": null,
                                "template": null
                            }, {
                                "bool": null,
                                "match_all": null,
                                "term": {
                                    "archived": {
                                        "value": false,
                                        "boost": null
                                    }
                                },
                                "wildcard": null,
                                "prefix": null,
                                "boosting": null,
                                "ids": null,
                                "custom_score": null,
                                "custom_filters_score": null,
                                "custom_boost_factor": null,
                                "constant_score": null,
                                "dis_max": null,
                                "filtered": null,
                                "multi_match": null,
                                "match": null,
                                "fuzzy": null,
                                "geo_shape": null,
                                "common": null,
                                "terms": null,
                                "range": null,
                                "regexp": null,
                                "has_child": null,
                                "has_parent": null,
                                "span_term": null,
                                "flt": null,
                                "simple_query_string": null,
                                "query_string": null,
                                "mlt": null,
                                "span_first": null,
                                "span_or": null,
                                "span_near": null,
                                "span_not": null,
                                "span_multi": null,
                                "top_children": null,
                                "nested": null,
                                "indices": null,
                                "function_score": null,
                                "template": null
                            }],
                            "must_not": null,
                            "should": null,
                            "minimum_should_match": null,
                            "disable_coord": null,
                            "boost": null
                        },
                        "match_all": null,
                        "term": null,
                        "wildcard": null,
                        "prefix": null,
                        "boosting": null,
                        "ids": null,
                        "custom_score": null,
                        "custom_filters_score": null,
                        "custom_boost_factor": null,
                        "constant_score": null,
                        "dis_max": null,
                        "filtered": null,
                        "multi_match": null,
                        "match": null,
                        "fuzzy": null,
                        "geo_shape": null,
                        "common": null,
                        "terms": null,
                        "range": null,
                        "regexp": null,
                        "has_child": null,
                        "has_parent": null,
                        "span_term": null,
                        "flt": null,
                        "simple_query_string": null,
                        "query_string": null,
                        "mlt": null,
                        "span_first": null,
                        "span_or": null,
                        "span_near": null,
                        "span_not": null,
                        "span_multi": null,
                        "top_children": null,
                        "nested": null,
                        "indices": null,
                        "function_score": null,
                        "template": null
                    },
                    "_cache": null,
                    "_name": null,
                    "_cache_key": null
                },
                "and": null,
                "or": null,
                "not": null,
                "script": null,
                "nested": null,
                "regexp": null
            }
        },
        "multi_match": null,
        "match": null,
        "fuzzy": null,
        "geo_shape": null,
        "common": null,
        "terms": null,
        "range": null,
        "regexp": null,
        "has_child": null,
        "has_parent": null,
        "span_term": null,
        "flt": null,
        "simple_query_string": null,
        "query_string": null,
        "mlt": null,
        "span_first": null,
        "span_or": null,
        "span_near": null,
        "span_not": null,
        "span_multi": null,
        "top_children": null,
        "nested": null,
        "indices": null,
        "function_score": null,
        "template": null
    },
    "filter": null
}

In any case, it would be nice to set this per request. Is there anything already available that does this?

[michaelbudnik]

I have similar problem which was raised on SO already: http://stackoverflow.com/questions/27918354/elasticsearch-nest-api-update-value-to-null

I would like to clear a property with an update but cannot.
What's more there are scenarios where I would like null values to be ignored (like it is working now) but in other scenarios I would like them to be stored. I would like to set it as per request basis.

Maybe allow to provide a custom type converter on per request basis?

[Mpdreamz]

Rather then changing how null should be serialised for the entire request the safest and most isolated way to do this is to introduce a separate POCO for the update where the property you want to clear has the following attribute.

[JsonProperty(NullValueHandling = NullValueHandling.Include)]

I just committed an example that illustrates this:

Example POCO:
https://github.com/elasticsearch/elasticsearch-net/blob/develop/src/Tests/Nest.Tests.Unit/Core/Update/UpdateTests.cs#L17-L25

Example update using that POCO:
https://github.com/elasticsearch/elasticsearch-net/blob/develop/src/Tests/Nest.Tests.Unit/Core/Update/UpdateTests.cs#L70-L81

which will generate the following JSON:
https://github.com/elasticsearch/elasticsearch-net/blob/develop/src/Tests/Nest.Tests.Unit/Core/Update/UpdateUsingPartialWithNull.json

[chrisdrobison]

That would work if I knew exactly which fields I were updating all the time. In my case I don't, which is why I'm using a dictionary. How do you accomplish the same thing with a dictionary without changing the setting for the entire request?

[Mpdreamz]

If its a nested property set a custom jsonconverter for that Dictionary Property e.g: https://github.com/elasticsearch/elasticsearch-net/blob/develop/src/Nest/Resolvers/Converters/DictionaryKeysAreNotPropertyNamesJsonConverter.cs

If you use a dictionary as the update type you might need to create a custom dictionary type and set a custom jsonconverter on that type.

Can you show us an example of how you are updating?

We might be able to handle dictionaries smarter for updates out of the box and ship with a dictionary json converter that keeps null values out of the box too.

[gmarz]

We might be able to handle dictionaries smarter for updates out of the box and ship with a dictionary json converter that keeps null values out of the box too.

+1 but we would still need to provide a way to toggle the behavior. I don't think the default should be to include nulls.

Maybe something like...

    .IdFrom(originalProject)
    .Doc(partialUpdate)
    .IncludeNullDictionaryValues();

[chrisdrobison]

To tell you what we are doing, we tail a MongoDB oplog and update ElasticSearch accordingly. Lots of changes in Mongo occur in a partial document fashion. So we translate those changes into a dictionary since maintaining POCOs for every kind of partial change would become unmanageable. Some changes set fields to null and we need those changes to occur in ElasticSearch.