bucket_selector not working with bucket_script or derivative as variable

[aurimasplu]

Elasticsearch version : 6.1.1-1

Plugins installed: X-pack

JVM version : java-1.8.0-openjdk-1.8.0.151-5.b12.el7_4.x86_64

OS version : RHEL 7.4

Description of the problem including expected versus actual behavior:
I want to use bucket_selector to remove buckets from the output based on the value generated by bucket_script. But it seems that bucket_selector does not find values in script. Using avg from same bucket works fine. Using derivative in bucket_script also works fine, which I suppose should be the same because derivative, bucket_script and bucket_selector all are parent pipeline aggregations.
Using derivative in bucket_selector also is not working and produces same error.

Steps to reproduce:

I make this query:

{
	"size": 0,
	"_source": [],
	"query": {
		"bool": {
			"filter": [{
					"range": {
						"@timestamp": {
							"gte": "1522850000000",
							"lte": "1522850400000",
							"format": "epoch_millis"
						}
					}
				}, {
					"query_string": {
						"analyze_wildcard": true,
						"query": "_exists_:interface.ifHCInOctets"
					}
				}
			]
		}
	},
	"aggs": {
		"DBF_Device": {
			"terms": {
				"field": "tag.agent_host",
				"size": 100,
				"order": {
					"_term": "desc"
				}
			},
			"aggs": {
				"DBF_Interface": {
					"terms": {
						"field": "tag.ifDescr",
						"size": 100,
						"order": {
							"_term": "desc"
						}
					},
					"aggs": {
						"DBF_Metric": {
							"date_histogram": {
								"interval": "5m",
								"field": "@timestamp"
							},
							"aggs": {
								"DBF_Metric_AVG": {
									"avg": {
										"field": "interface.ifHCInOctets",
										"script": {
											"inline": "(_value*8)/300"
										}
									}
								},
								"DBF_Metric_DER": {
									"derivative": {
										"buckets_path": "DBF_Metric_AVG"
									}
								},
								"DBF_Speed": {
									"max": {
										"field": "interface.ifHighSpeed",
										"script": {
											"inline": "_value*1000000"
										}
									}
								},
								"DBF_Metric_Percent": {
									"bucket_script": {
										"buckets_path": {
											"my_var1": "DBF_Metric_DER",
											"my_var2": "DBF_Speed"
										},
										"script": "(params.my_var1 * 100)/ params.my_var2"
									}
								},
								"DBF_Bucket_filter": {
										"bucket_selector": {
											"buckets_path": {
												"my_var3": "DBF_Metric_Percent"
											},
											"script": "params.my_var3 > 100000000"
										}
									}
							}
						}
					}
				}
			}
		}
	}
}

And I get this output:

{
    "error": {
        "root_cause": [],
        "type": "search_phase_execution_exception",
        "reason": "",
        "phase": "fetch",
        "grouped": true,
        "failed_shards": [],
        "caused_by": {
            "type": "script_exception",
            "reason": "runtime error",
            "script_stack": [
                "params.my_var3 > 100000000",
                "      ^---- HERE"
            ],
            "script": "params.my_var3 > 100000000",
            "lang": "painless",
            "caused_by": {
                "type": "null_pointer_exception",
                "reason": null
            }
        }
    },
    "status": 503
}

Query works fine if I remove bucket_selector part. If in bucket_selector I replace DBF_Metric_Percent with DBF_Speed query also works fine.

[colings86]

@aurimasplu I reproduced your issues using this following script:

PUT test
{
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  },
  "mappings": {
    "doc": {
      "properties": {
        "date": {
          "type": "date"
        },
        "number": {
          "type": "long"
        }
      }
    }
  }
}

PUT test/doc/1
{
  "date": "2018-01-01",
  "number": 1
}
PUT test/doc/2
{
  "date": "2018-01-02",
  "number": 2
}
PUT test/doc/3
{
  "date": "2018-01-03",
  "number": 3
}
PUT test/doc/5
{
  "date": "2018-01-05",
  "number": 5
}

GET test/_search
{
  "size": 0,
  "aggs": {
    "histo": {
      "date_histogram": {
        "field": "date",
        "interval": "day"
      },
      "aggs": {
        "avg": {
          "avg": {
            "field": "number"
          }
        },
        "script": {
          "bucket_script": {
            "buckets_path": {
              "my_var1": "avg"
            },
            "script": "params.my_var1 * 2"
          }
        },
        "selector": {
          "bucket_selector": {
            "buckets_path": {
              "my_var3": "script"
            },
            "script": "params.my_var3 > 4"
          }
        }
      }
    }
  }
}

Notice that there is no document for 2018-01-04 which means the date histogram has an empty bucket. There is a known issue (#27377) which means the bucket_script aggregation is not run on buckets that have a doc_count of 0 due to how pipeline aggregations deal with gaps in data. The same issue is happening here where empty buckets do not have a value for the bucket_script because its not run so the bucket_selector is throwing a null pointer when it tries to unbox the null value to compare it in the condition. As a work around you can add a default value for params.my_var3 to your bucket_selector script so it can evaluate the expression even when the bucket_script agg is not run. For my example above that would look like the following:

GET test/_search
{
  "size": 0,
  "aggs": {
    "histo": {
      "date_histogram": {
        "field": "date",
        "interval": "day"
      },
      "aggs": {
        "avg": {
          "avg": {
            "field": "number"
          }
        },
        "script": {
          "bucket_script": {
            "buckets_path": {
              "my_var1": "avg"
            },
            "script": "params.my_var1 * 2"
          }
        },
        "selector": {
          "bucket_selector": {
            "buckets_path": {
              "my_var3": "script"
            },
            "script": "(params.my_var3 ?: 0) > 4"
          }
        }
      }
    }
  }
}

Note the ?: 0 which means that if params.my_var3 is null 0 will be used instead.

I'm going to close this issue in favour of #27377 since this is the same problem that #27377 describes.