From 7724e06b0516519bad061cd16048670ab7b8a86a Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Fri, 18 May 2018 20:08:41 -0400 Subject: [PATCH 01/22] NIFI-5214 Added REST LookupService --- .../nifi-lookup-services/pom.xml | 58 +++++ .../apache/nifi/lookup/RestLookupService.java | 245 ++++++++++++++++++ ...g.apache.nifi.controller.ControllerService | 1 + .../additionalDetails.html | 42 +++ .../nifi/lookup/MockRestLookupService.groovy | 30 +++ .../nifi/lookup/TestRestLookupService.groovy | 143 ++++++++++ .../TestRestLookupServiceProcessor.groovy | 45 ++++ 7 files changed, 564 insertions(+) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs.org.apache.nifi.lookup.RestLookupService/additionalDetails.html create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/MockRestLookupService.groovy create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupServiceProcessor.groovy diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml index dbcfa8d7db16..7b4ea087e432 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml @@ -67,6 +67,11 @@ + + com.squareup.okhttp3 + okhttp + 3.10.0 + org.apache.nifi nifi-mock @@ -83,6 +88,28 @@ junit test + + org.apache.nifi + nifi-record-serialization-service-api + compile + + + org.apache.nifi + nifi-ssl-context-service-api + compile + + + org.apache.nifi + nifi-record-path + 1.7.0-SNAPSHOT + compile + + + org.apache.nifi + nifi-mock-record-utils + 1.7.0-SNAPSHOT + test + @@ -98,6 +125,37 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + + compile + testCompile + + + groovy-eclipse-compiler + + + + + 1.8 + 1.8 + + + + org.codehaus.groovy + groovy-eclipse-compiler + 2.9.2-01 + + + org.codehaus.groovy + groovy-eclipse-batch + 2.4.3-01 + + + diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java new file mode 100644 index 000000000000..edd1d37c108a --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.lookup; + +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.Validator; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.record.path.FieldValue; +import org.apache.nifi.record.path.RecordPath; +import org.apache.nifi.schema.access.SchemaNotFoundException; +import org.apache.nifi.serialization.MalformedRecordException; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.MapRecord; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.ssl.SSLContextService; +import org.apache.nifi.util.StringUtils; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +@Tags({ "rest", "lookup", "json", "xml" }) +@CapabilityDescription("Use a REST service to enrich records.") +public class RestLookupService extends AbstractControllerService implements LookupService { + static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder() + .name("rest-lookup-record-reader") + .displayName("Record Reader") + .description("The record reader to use for loading the payload and handling it as a record set.") + .expressionLanguageSupported(ExpressionLanguageScope.NONE) + .identifiesControllerService(RecordReaderFactory.class) + .addValidator(Validator.VALID) + .required(true) + .build(); + + static final PropertyDescriptor RECORD_PATH = new PropertyDescriptor.Builder() + .name("rest-lookup-record-path") + .displayName("Record Path") + .description("An optional record path that can be used to define where in a record to get the real data to merge " + + "into the record set to be enriched. See documentation for examples of when this might be useful.") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .addValidator(Validator.VALID) + .required(false) + .build(); + + static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("rest-lookup-ssl-context-service") + .displayName("SSL Context Service") + .description("The SSL Context Service used to provide client certificate information for TLS/SSL " + + "connections.") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + static final String ENDPOINT_KEY = "endpoint"; + static final String MIME_TYPE_KEY = "mime.type"; + static final String BODY_KEY = "request.body"; + static final String METHOD_KEY = "request.method"; + + static final List DESCRIPTORS; + + static { + DESCRIPTORS = Collections.unmodifiableList(Arrays.asList( + RECORD_READER, + RECORD_PATH, + SSL_CONTEXT_SERVICE + )); + } + + protected List getSupportedPropertyDescriptors() { + return DESCRIPTORS; + } + + private SSLContextService sslContextService; + private RecordReaderFactory readerFactory; + private RecordPath recordPath; + private OkHttpClient client; + + @OnEnabled + public void onEnabled(final ConfigurationContext context) { + sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class); + + OkHttpClient.Builder builder = new OkHttpClient.Builder(); + final SSLContextService sslService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + final SSLContext sslContext = sslService == null ? null : sslService.createSSLContext(SSLContextService.ClientAuth.WANT); + if (sslService != null) { + builder.sslSocketFactory(sslContext.getSocketFactory()); + } + + client = builder.build(); + + String path = context.getProperty(RECORD_PATH).isSet() ? context.getProperty(RECORD_PATH).getValue() : null; + if (!StringUtils.isBlank(path)) { + recordPath = RecordPath.compile(path); + } + } + + @Override + public Optional lookup(Map coordinates) throws LookupFailureException { + final String endpoint = (String)coordinates.get(ENDPOINT_KEY); + final String mimeType = (String)coordinates.get(MIME_TYPE_KEY); + final String method = (String)coordinates.get(METHOD_KEY); + final String body = (String)coordinates.get(BODY_KEY); + + if (StringUtils.isBlank(body) && (method.equals("post") || method.equals("put"))) { + throw new LookupFailureException( + String.format("Used HTTP verb %s without specifying the %s key to provide a payload.", method, BODY_KEY) + ); + } + + Request request = buildRequest(mimeType, method, body, endpoint); + try { + Response response = executeRequest(request); + InputStream is = response.body().byteStream(); + + response.close(); + + Record record = handleResponse(is, coordinates); + + return Optional.of(record); + } catch (MalformedRecordException | SchemaNotFoundException | IOException e) { + getLogger().error("Could not execute lookup.", e); + throw new LookupFailureException(e); + } + } + + protected Response executeRequest(Request request) throws IOException { + return client.newCall(request).execute(); + } + + private Record handleResponse(InputStream is, Map coordinates) throws SchemaNotFoundException, MalformedRecordException, IOException { + Map variables = coordinates.entrySet().stream() + .collect(Collectors.toMap( + e -> e.getKey(), + e -> e.getValue().toString() + )); + RecordReader reader = readerFactory.createRecordReader(variables, is, getLogger()); + Record record = reader.nextRecord(); + + if (recordPath != null) { + Optional fv = recordPath.evaluate(record).getSelectedFields().findFirst(); + if (fv.isPresent()) { + FieldValue fieldValue = fv.get(); + RecordSchema schema = new SimpleRecordSchema(Arrays.asList(fieldValue.getField())); + String[] parts = recordPath.getPath().split("/"); + String last = parts[parts.length - 1]; + + Record temp; + Object value = fieldValue.getValue(); + if (value instanceof Record) { + temp = (Record)value; + } else if (value instanceof Map) { + temp = new MapRecord(schema, (Map)value); + } else { + temp = new MapRecord(schema, new HashMap(){{ + put(last, value); + }}); + } + + record = temp; + } + } + + reader.close(); + + return record; + } + + private Request buildRequest(final String mimeType, final String method, final String body, final String endpoint) { + final MediaType mt = MediaType.parse(mimeType); + RequestBody requestBody = null; + if (body != null) { + requestBody = RequestBody.create(mt, body); + } + Request.Builder request = new Request.Builder() + .url(endpoint); + switch(method.toLowerCase()) { + case "delete": + request = body != null ? request.delete(requestBody) : request.delete(); + break; + case "get": + request = request.get(); + break; + case "post": + request = request.post(requestBody); + break; + case "put": + request = request.put(requestBody); + break; + } + + return request.build(); + } + + @Override + public Class getValueType() { + return Record.class; + } + + @Override + public Set getRequiredKeys() { + return new HashSet(){{ + add(ENDPOINT_KEY); + add(MIME_TYPE_KEY); + add(METHOD_KEY); + }}; + } +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService index 544ec91a9eb2..34395e8720bc 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService @@ -15,6 +15,7 @@ org.apache.nifi.lookup.maxmind.IPLookupService org.apache.nifi.lookup.CSVRecordLookupService org.apache.nifi.lookup.PropertiesFileLookupService +org.apache.nifi.lookup.RestLookupService org.apache.nifi.lookup.SimpleKeyValueLookupService org.apache.nifi.lookup.SimpleCsvFileLookupService org.apache.nifi.lookup.XMLFileLookupService diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs.org.apache.nifi.lookup.RestLookupService/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs.org.apache.nifi.lookup.RestLookupService/additionalDetails.html new file mode 100644 index 000000000000..1bba4e733f7e --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs.org.apache.nifi.lookup.RestLookupService/additionalDetails.html @@ -0,0 +1,42 @@ + + + + + + RestLookupService + + + +

This lookup service has the following required keys:

+
    +
  • endpoint
    • +
  • mime.type
    • +
  • request.method; valid values: +
      +
    • delete
      • +
    • get
      • +
    • post
      • +
    • put
      • +
    +
    • +
+

In addition to the required keys, a key "body" can be added which contains a string representing JSON, XML, etc. to be sent with any + of those methods except for "get."

+

The record reader is used to consume the response of the REST service call and turn it into one or more records. The record path property + is provided to allow for a lookup path to either a nested record or a single point deep in the REST response. Note: a valid schema must be + built that encapsulates the REST response accurately in order for this service to work.

+ + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/MockRestLookupService.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/MockRestLookupService.groovy new file mode 100644 index 000000000000..b6431e46d5c9 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/MockRestLookupService.groovy @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.lookup + +import okhttp3.Request +import okhttp3.Response + +class MockRestLookupService extends RestLookupService { + Response response + + @Override + protected Response executeRequest(Request request) { + return response + } +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy new file mode 100644 index 000000000000..8bdf3912946b --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.lookup + +import okhttp3.* +import org.apache.nifi.serialization.SimpleRecordSchema +import org.apache.nifi.serialization.record.MapRecord +import org.apache.nifi.serialization.record.MockRecordParser +import org.apache.nifi.serialization.record.RecordField +import org.apache.nifi.serialization.record.RecordFieldType +import org.apache.nifi.serialization.record.RecordSchema +import org.apache.nifi.util.TestRunner +import org.apache.nifi.util.TestRunners +import org.junit.Assert +import org.junit.Before +import org.junit.Test + +import static groovy.json.JsonOutput.toJson + +class TestRestLookupService { + TestRunner runner + MockRecordParser recordReader + MockRestLookupService lookupService + + static final String JSON_TYPE = "application/json" + + @Before + void setup() { + recordReader = new MockRecordParser() + lookupService = new MockRestLookupService() + runner = TestRunners.newTestRunner(TestRestLookupServiceProcessor.class) + runner.addControllerService("lookupService", lookupService) + runner.addControllerService("recordReader", recordReader) + runner.setProperty(lookupService, RestLookupService.RECORD_READER, "recordReader") + runner.setProperty("Lookup Service", "lookupService") + runner.enableControllerService(lookupService) + runner.enableControllerService(recordReader) + runner.assertValid() + } + + @Test + void testSimpleLookup() { + recordReader.addSchemaField("name", RecordFieldType.STRING) + recordReader.addSchemaField("age", RecordFieldType.INT) + recordReader.addSchemaField("sport", RecordFieldType.STRING) + + recordReader.addRecord("John Doe", 48, "Soccer") + recordReader.addRecord("Jane Doe", 47, "Tennis") + recordReader.addRecord("Sally Doe", 47, "Curling") + + lookupService.response = buildResponse(toJson([ simpleTest: true]), JSON_TYPE) + def result = lookupService.lookup(getCoordinates("http://localhost:8080", JSON_TYPE, "get")) + Assert.assertTrue(result.isPresent()) + def record = result.get() + Assert.assertEquals("John Doe", record.getAsString("name")) + Assert.assertEquals(48, record.getAsInt("age")) + Assert.assertEquals("Soccer", record.getAsString("sport")) + } + + @Test + void testNestedLookup() { + runner.disableControllerService(lookupService) + runner.setProperty(lookupService, RestLookupService.RECORD_PATH, "/person") + runner.enableControllerService(lookupService) + runner.assertValid() + + recordReader.addSchemaField("id", RecordFieldType.INT) + final List personFields = new ArrayList<>() + final RecordField nameField = new RecordField("name", RecordFieldType.STRING.getDataType()) + final RecordField ageField = new RecordField("age", RecordFieldType.INT.getDataType()) + final RecordField sportField = new RecordField("sport", RecordFieldType.STRING.getDataType()) + personFields.add(nameField) + personFields.add(ageField) + personFields.add(sportField) + final RecordSchema personSchema = new SimpleRecordSchema(personFields) + recordReader.addSchemaField("person", RecordFieldType.RECORD) + recordReader.addRecord(1, new MapRecord(personSchema, new HashMap() {{ + put("name", "John Doe") + put("age", 48) + put("sport", "Soccer") + }})) + + lookupService.response = buildResponse(toJson([ simpleTest: true]), JSON_TYPE) + def result = lookupService.lookup(getCoordinates("http://localhost:8080", JSON_TYPE, "get")) + Assert.assertTrue(result.isPresent()) + def record = result.get() + + Assert.assertEquals("John Doe", record.getAsString("name")) + Assert.assertEquals(48, record.getAsInt("age")) + Assert.assertEquals("Soccer", record.getAsString("sport")) + + /* + * Test deep lookup + */ + + runner.disableControllerService(lookupService) + runner.setProperty(lookupService, RestLookupService.RECORD_PATH, "/person/sport") + runner.enableControllerService(lookupService) + runner.assertValid() + + result = lookupService.lookup(getCoordinates("http://localhost:8080", JSON_TYPE, "get")) + Assert.assertTrue(result.isPresent()) + record = result.get() + Assert.assertNotNull(record.getAsString("sport")) + Assert.assertEquals("Soccer", record.getAsString("sport")) + } + + private Map getCoordinates(String endpoint, String mimeType, String method) { + def retVal = [:] + retVal[RestLookupService.ENDPOINT_KEY] = endpoint + retVal[RestLookupService.MIME_TYPE_KEY] = mimeType + retVal[RestLookupService.METHOD_KEY] = method + + retVal + } + + private Response buildResponse(String resp, String mimeType) { + return new Response.Builder() + .code(200) + .body( + ResponseBody.create(MediaType.parse(mimeType), resp) + ) + .message("Test") + .protocol(Protocol.HTTP_1_1) + .request(new Request.Builder().url("http://localhost:8080").get().build()) + .build() + } +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupServiceProcessor.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupServiceProcessor.groovy new file mode 100644 index 000000000000..9ca24421b65e --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupServiceProcessor.groovy @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.lookup + +import org.apache.nifi.components.PropertyDescriptor +import org.apache.nifi.processor.AbstractProcessor +import org.apache.nifi.processor.ProcessContext +import org.apache.nifi.processor.ProcessSession +import org.apache.nifi.processor.exception.ProcessException + +class TestRestLookupServiceProcessor extends AbstractProcessor { + static final PropertyDescriptor CLIENT_SERVICE = new PropertyDescriptor.Builder() + .name("Lookup Service") + .description("RestLookupService") + .identifiesControllerService(RestLookupService.class) + .required(true) + .build() + + @Override + void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + + } + + @Override + protected List getSupportedPropertyDescriptors() { + List propDescs = new ArrayList<>() + propDescs.add(CLIENT_SERVICE) + return propDescs + } +} From 8584d17d68aabeeb71ec8d199b992bba0804fcdc Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Sun, 20 May 2018 18:46:08 -0400 Subject: [PATCH 02/22] NIFI-5214 Added support for the new ProxyConfigurationService --- .../nifi-lookup-services/pom.xml | 4 ++ .../apache/nifi/lookup/RestLookupService.java | 37 ++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml index 7b4ea087e432..9283c9ce7a2e 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml @@ -110,6 +110,10 @@ 1.7.0-SNAPSHOT test + + org.apache.nifi + nifi-proxy-configuration-api + diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index edd1d37c108a..bdac938b5b43 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -17,6 +17,7 @@ package org.apache.nifi.lookup; +import okhttp3.Credentials; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Request; @@ -30,6 +31,9 @@ import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.proxy.ProxyConfiguration; +import org.apache.nifi.proxy.ProxyConfigurationService; +import org.apache.nifi.proxy.ProxySpec; import org.apache.nifi.record.path.FieldValue; import org.apache.nifi.record.path.RecordPath; import org.apache.nifi.schema.access.SchemaNotFoundException; @@ -46,6 +50,7 @@ import javax.net.ssl.SSLContext; import java.io.IOException; import java.io.InputStream; +import java.net.Proxy; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -88,6 +93,10 @@ public class RestLookupService extends AbstractControllerService implements Look .identifiesControllerService(SSLContextService.class) .build(); + private static final ProxySpec[] PROXY_SPECS = {ProxySpec.HTTP_AUTH, ProxySpec.SOCKS}; + public static final PropertyDescriptor PROXY_CONFIGURATION_SERVICE + = ProxyConfiguration.createProxyConfigPropertyDescriptor(true, PROXY_SPECS); + static final String ENDPOINT_KEY = "endpoint"; static final String MIME_TYPE_KEY = "mime.type"; static final String BODY_KEY = "request.body"; @@ -99,7 +108,8 @@ public class RestLookupService extends AbstractControllerService implements Look DESCRIPTORS = Collections.unmodifiableList(Arrays.asList( RECORD_READER, RECORD_PATH, - SSL_CONTEXT_SERVICE + SSL_CONTEXT_SERVICE, + PROXY_CONFIGURATION_SERVICE )); } @@ -107,6 +117,7 @@ protected List getSupportedPropertyDescriptors() { return DESCRIPTORS; } + private ProxyConfigurationService proxyConfigurationService; private SSLContextService sslContextService; private RecordReaderFactory readerFactory; private RecordPath recordPath; @@ -116,8 +127,15 @@ protected List getSupportedPropertyDescriptors() { public void onEnabled(final ConfigurationContext context) { sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class); + proxyConfigurationService = context.getProperty(PROXY_CONFIGURATION_SERVICE) + .asControllerService(ProxyConfigurationService.class); OkHttpClient.Builder builder = new OkHttpClient.Builder(); + + if (proxyConfigurationService != null) { + setProxy(builder); + } + final SSLContextService sslService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); final SSLContext sslContext = sslService == null ? null : sslService.createSSLContext(SSLContextService.ClientAuth.WANT); if (sslService != null) { @@ -132,6 +150,23 @@ public void onEnabled(final ConfigurationContext context) { } } + private void setProxy(OkHttpClient.Builder builder) { + ProxyConfiguration config = proxyConfigurationService.getConfiguration(); + if (!config.getProxyType().equals(Proxy.Type.DIRECT)) { + final Proxy proxy = config.createProxy(); + builder.proxy(proxy); + + if (config.hasCredential()){ + builder.proxyAuthenticator((route, response) -> { + final String credential= Credentials.basic(config.getProxyUserName(), config.getProxyUserPassword()); + return response.request().newBuilder() + .header("Proxy-Authorization", credential) + .build(); + }); + } + } + } + @Override public Optional lookup(Map coordinates) throws LookupFailureException { final String endpoint = (String)coordinates.get(ENDPOINT_KEY); From b1fcac90b6bb7bf890d3efb9976ea8474524dda1 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Mon, 21 May 2018 06:30:06 -0400 Subject: [PATCH 03/22] NIFI-5214 Integration tests added. --- .../nifi-standard-processors/pom.xml | 6 + .../nifi/processors/standard/TestGetHTTP.java | 1 + .../processors/standard/TestInvokeHTTP.java | 1 + .../standard/TestInvokeHttpSSL.java | 1 + .../processors/standard/TestPostHTTP.java | 1 + .../standard/util/TestInvokeHttpCommon.java | 2 +- .../org/apache/nifi/web/util}/TestServer.java | 2 +- nifi-nar-bundles/nifi-standard-bundle/pom.xml | 1 + .../nifi-lookup-services/pom.xml | 27 ++++ .../apache/nifi/lookup/RestLookupService.java | 16 +- .../nifi/lookup/RestLookupServiceIT.groovy | 138 ++++++++++++++++++ .../nifi/lookup/TestRestLookupService.groovy | 1 + .../{ => rest}/MockRestLookupService.groovy | 3 +- .../apache/nifi/lookup/rest/SchemaUtil.groovy | 57 ++++++++ .../lookup/rest/handlers/ComplexJson.groovy | 28 ++++ .../lookup/rest/handlers/SimpleJson.groovy | 20 +++ .../rest/handlers/SimpleJsonArray.groovy | 25 ++++ 17 files changed, 318 insertions(+), 12 deletions(-) rename nifi-nar-bundles/nifi-standard-bundle/{nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard => nifi-standard-web-utils/src/main/java/org/apache/nifi/web/util}/TestServer.java (99%) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy rename nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/{ => rest}/MockRestLookupService.groovy (92%) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJsonArray.groovy diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index babc43825d96..6d084d7c161e 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -321,6 +321,12 @@ org.apache.nifi nifi-schema-registry-service-api + + org.apache.nifi + nifi-standard-web-utils + 1.7.0-SNAPSHOT + test + diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java index 3e33948d6384..e1d76e02428c 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java @@ -18,6 +18,7 @@ import org.apache.nifi.components.state.Scope; import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.web.util.TestServer; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.ssl.SSLContextService; import org.apache.nifi.ssl.StandardSSLContextService; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java index bb4f7ee00b9e..70a183ab3e0a 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java @@ -17,6 +17,7 @@ package org.apache.nifi.processors.standard; import org.apache.nifi.processors.standard.util.TestInvokeHttpCommon; +import org.apache.nifi.web.util.TestServer; import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunners; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java index 7f0bcbb8276b..43380aba4c5a 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHttpSSL.java @@ -18,6 +18,7 @@ package org.apache.nifi.processors.standard; import org.apache.nifi.processors.standard.util.TestInvokeHttpCommon; +import org.apache.nifi.web.util.TestServer; import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.util.TestRunners; import org.junit.After; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPostHTTP.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPostHTTP.java index abb19518b78d..ef3448735b65 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPostHTTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestPostHTTP.java @@ -31,6 +31,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.nifi.flowfile.attributes.CoreAttributes; +import org.apache.nifi.web.util.TestServer; import org.apache.nifi.ssl.SSLContextService; import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.util.FlowFileUnpackagerV3; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java index 5242b8f9dff4..3304e6267788 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestInvokeHttpCommon.java @@ -19,11 +19,11 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processors.standard.InvokeHTTP; -import org.apache.nifi.processors.standard.TestServer; import org.apache.nifi.provenance.ProvenanceEventRecord; import org.apache.nifi.provenance.ProvenanceEventType; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; +import org.apache.nifi.web.util.TestServer; import org.eclipse.jetty.security.ConstraintSecurityHandler; import org.eclipse.jetty.security.DefaultIdentityService; import org.eclipse.jetty.security.HashLoginService; diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestServer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/src/main/java/org/apache/nifi/web/util/TestServer.java similarity index 99% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestServer.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/src/main/java/org/apache/nifi/web/util/TestServer.java index 4410ca50fc7a..47bbdeacf8da 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestServer.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/src/main/java/org/apache/nifi/web/util/TestServer.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.nifi.processors.standard; +package org.apache.nifi.web.util; import java.util.Map; import org.apache.nifi.ssl.StandardSSLContextService; diff --git a/nifi-nar-bundles/nifi-standard-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/pom.xml index 0cadde34f216..acfb6cdbdce0 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/pom.xml @@ -30,6 +30,7 @@ nifi-standard-nar nifi-jolt-transform-json-ui nifi-standard-utils + nifi-standard-web-utils 2.9.5 diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml index 9283c9ce7a2e..0fca07bc5d70 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml @@ -114,6 +114,33 @@ org.apache.nifi nifi-proxy-configuration-api + + org.apache.nifi + nifi-record-serialization-services + 1.7.0-SNAPSHOT + test + + + org.apache.nifi + nifi-schema-registry-service-api + test + + + javax.servlet + javax.servlet-api + test + + + org.apache.nifi + nifi-standard-web-utils + 1.7.0-SNAPSHOT + test + + + org.eclipse.jetty + jetty-servlet + test + diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index bdac938b5b43..efacfbbea425 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -85,13 +85,13 @@ public class RestLookupService extends AbstractControllerService implements Look .build(); static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() - .name("rest-lookup-ssl-context-service") - .displayName("SSL Context Service") - .description("The SSL Context Service used to provide client certificate information for TLS/SSL " - + "connections.") - .required(false) - .identifiesControllerService(SSLContextService.class) - .build(); + .name("rest-lookup-ssl-context-service") + .displayName("SSL Context Service") + .description("The SSL Context Service used to provide client certificate information for TLS/SSL " + + "connections.") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); private static final ProxySpec[] PROXY_SPECS = {ProxySpec.HTTP_AUTH, ProxySpec.SOCKS}; public static final PropertyDescriptor PROXY_CONFIGURATION_SERVICE @@ -185,8 +185,6 @@ public Optional lookup(Map coordinates) throws LookupFai Response response = executeRequest(request); InputStream is = response.body().byteStream(); - response.close(); - Record record = handleResponse(is, coordinates); return Optional.of(record); diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy new file mode 100644 index 000000000000..db8516a41f3a --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -0,0 +1,138 @@ +package org.apache.nifi.lookup + +import org.apache.avro.Schema +import org.apache.nifi.avro.AvroTypeUtil +import org.apache.nifi.json.JsonTreeReader +import org.apache.nifi.lookup.rest.SchemaUtil +import org.apache.nifi.lookup.rest.handlers.ComplexJson +import org.apache.nifi.lookup.rest.handlers.SimpleJson +import org.apache.nifi.lookup.rest.handlers.SimpleJsonArray +import org.apache.nifi.schema.access.SchemaAccessUtils +import org.apache.nifi.serialization.record.MockSchemaRegistry +import org.apache.nifi.serialization.record.Record +import org.apache.nifi.serialization.record.RecordSchema +import org.apache.nifi.util.TestRunner +import org.apache.nifi.util.TestRunners +import org.apache.nifi.web.util.TestServer +import org.eclipse.jetty.servlet.ServletHandler +import org.junit.Assert +import org.junit.Before +import org.junit.Test + +class RestLookupServiceIT { + static final JsonTreeReader reader + static final MockSchemaRegistry registry = new MockSchemaRegistry() + static final RecordSchema simpleSchema + static final RecordSchema nestedSchema + + TestRunner runner + RestLookupService lookupService + + static { + simpleSchema = AvroTypeUtil.createSchema(new Schema.Parser().parse(SchemaUtil.SIMPLE)) + nestedSchema = AvroTypeUtil.createSchema(new Schema.Parser().parse(SchemaUtil.COMPLEX)) + registry.addSchema("simple", simpleSchema) + registry.addSchema("complex", nestedSchema) + + reader = new JsonTreeReader() + } + + @Before + void setup() { + lookupService = new RestLookupService() + + runner = TestRunners.newTestRunner(TestRestLookupServiceProcessor.class) + runner.addControllerService("jsonReader", reader) + runner.addControllerService("registry", registry) + runner.addControllerService("lookupService", lookupService) + runner.setProperty(reader, SchemaAccessUtils.SCHEMA_REGISTRY, "registry") + runner.setProperty(lookupService, SchemaAccessUtils.SCHEMA_REGISTRY, "registry") + runner.setProperty(lookupService, RestLookupService.RECORD_READER, "jsonReader") + runner.setProperty(TestRestLookupServiceProcessor.CLIENT_SERVICE, "lookupService") + runner.enableControllerService(registry) + runner.enableControllerService(reader) + runner.enableControllerService(lookupService) + + runner.assertValid() + } + + @Test + void simpleJson() { + TestServer server = new TestServer() + ServletHandler handler = new ServletHandler() + handler.addServletWithMapping(SimpleJson.class, "/simple") + server.addHandler(handler) + try { + server.startServer() + def coordinates = [ + "schema.name": "simple", + "endpoint": server.url + "/simple", + "mime.type": "application/json", + "request.method": "get" + ] + + Optional response = lookupService.lookup(coordinates) + Assert.assertTrue(response.isPresent()) + def record = response.get() + Assert.assertEquals("john.smith", record.getAsString("username")) + Assert.assertEquals("testing1234", record.getAsString("password")) + } finally { + server.shutdownServer() + } + } + + @Test + void simpleJsonArray() { + TestServer server = new TestServer() + ServletHandler handler = new ServletHandler() + handler.addServletWithMapping(SimpleJsonArray.class, "/simple_array") + server.addHandler(handler) + try { + server.startServer() + def coordinates = [ + "schema.name": "simple", + "endpoint": server.url + "/simple_array", + "mime.type": "application/json", + "request.method": "get" + ] + + Optional response = lookupService.lookup(coordinates) + Assert.assertTrue(response.isPresent()) + def record = response.get() + Assert.assertEquals("john.smith", record.getAsString("username")) + Assert.assertEquals("testing1234", record.getAsString("password")) + } finally { + server.shutdownServer() + } + } + + @Test + void complexJson() { + runner.disableControllerService(lookupService) + runner.setProperty(lookupService, RestLookupService.RECORD_PATH, "/top/middle/inner") + runner.enableControllerService(lookupService) + + TestServer server = new TestServer() + ServletHandler handler = new ServletHandler() + handler.addServletWithMapping(ComplexJson.class, "/complex") + server.addHandler(handler) + try { + server.startServer() + def coordinates = [ + "schema.name": "complex", + "endpoint": server.url + "/complex", + "mime.type": "application/json", + "request.method": "get" + ] + + Optional response = lookupService.lookup(coordinates) + Assert.assertTrue(response.isPresent()) + def record = response.get() + Assert.assertEquals("jane.doe", record.getAsString("username")) + Assert.assertEquals("testing7890", record.getAsString("password")) + Assert.assertEquals("jane.doe@company.com", record.getAsString("email")) + } finally { + server.shutdownServer() + } + } +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy index 8bdf3912946b..aac1f9671209 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy @@ -18,6 +18,7 @@ package org.apache.nifi.lookup import okhttp3.* +import org.apache.nifi.lookup.rest.MockRestLookupService import org.apache.nifi.serialization.SimpleRecordSchema import org.apache.nifi.serialization.record.MapRecord import org.apache.nifi.serialization.record.MockRecordParser diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/MockRestLookupService.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/MockRestLookupService.groovy similarity index 92% rename from nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/MockRestLookupService.groovy rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/MockRestLookupService.groovy index b6431e46d5c9..210f329fc86c 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/MockRestLookupService.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/MockRestLookupService.groovy @@ -15,10 +15,11 @@ * limitations under the License. */ -package org.apache.nifi.lookup +package org.apache.nifi.lookup.rest import okhttp3.Request import okhttp3.Response +import org.apache.nifi.lookup.RestLookupService class MockRestLookupService extends RestLookupService { Response response diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy new file mode 100644 index 000000000000..49ece37db2dc --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy @@ -0,0 +1,57 @@ +package org.apache.nifi.lookup.rest + +class SchemaUtil { + static final String SIMPLE = """ + { + "type": "record", + "name": "SimpleRecord", + "fields": [ + { + "name": "username", + "type": "string" + }, + { + "name": "password", + "type": "string" + } + ] + } + """ + + static final String COMPLEX = """{ + "type": "record", + "name": "ComplexRecord", + "fields": [ + { + "name": "top", + "type": { + "type": "record", + "name": "TopRecord", + "fields": [ + { + "name": "middle", + "type": { + "name": "MiddleRecord", + "type": "record", + "fields": [ + { + "name": "inner", + "type": { + "type": "record", + "name": "InnerRecord", + "fields": [ + { "name": "username", "type": "string" }, + { "name": "password", "type": "string" }, + { "name": "email", "type": "string" } + ] + } + } + ] + } + } + ] + } + } + ] + }""" +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy new file mode 100644 index 000000000000..1e714781bef1 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy @@ -0,0 +1,28 @@ +package org.apache.nifi.lookup.rest.handlers + +import javax.servlet.http.HttpServlet +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +import static groovy.json.JsonOutput.prettyPrint +import static groovy.json.JsonOutput.toJson + +class ComplexJson extends HttpServlet { + @Override + void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + response.contentType = "application/json" + response.outputStream.write(prettyPrint( + toJson([ + top: [ + middle: [ + inner: [ + "username": "jane.doe", + "password": "testing7890", + "email": "jane.doe@company.com" + ] + ] + ] + ]) + ).bytes) + } +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy new file mode 100644 index 000000000000..eb17a7baa36a --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy @@ -0,0 +1,20 @@ +package org.apache.nifi.lookup.rest.handlers + +import static groovy.json.JsonOutput.* + +import javax.servlet.http.HttpServlet +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +class SimpleJson extends HttpServlet { + @Override + void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + response.contentType = "application/json" + response.outputStream.write(prettyPrint( + toJson([ + username: "john.smith", + password: "testing1234" + ]) + ).bytes) + } +} diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJsonArray.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJsonArray.groovy new file mode 100644 index 000000000000..084525999802 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJsonArray.groovy @@ -0,0 +1,25 @@ +package org.apache.nifi.lookup.rest.handlers + +import javax.servlet.http.HttpServlet +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +import static groovy.json.JsonOutput.prettyPrint +import static groovy.json.JsonOutput.toJson + +class SimpleJsonArray extends HttpServlet { + @Override + void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + response.contentType = "application/json" + response.outputStream.write(prettyPrint( + toJson([[ + username: "john.smith", + password: "testing1234" + ], + [ + username: "jane.doe", + password: "testing7890" + ]]) + ).bytes) + } +} From f4d3493c1cfbc062bfd7d10d6a6c2f4092f1d2f6 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Mon, 21 May 2018 08:54:39 -0400 Subject: [PATCH 04/22] NIFI-5214 Added missing pom.xml and a change from a code review. --- .../nifi-standard-web-utils/pom.xml | 39 +++++++++++++++++++ .../apache/nifi/lookup/RestLookupService.java | 2 - 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/pom.xml diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/pom.xml new file mode 100644 index 000000000000..47774e87cf72 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/pom.xml @@ -0,0 +1,39 @@ + + + + + nifi-standard-bundle + org.apache.nifi + 1.7.0-SNAPSHOT + + 4.0.0 + nifi-standard-web-utils + + + org.apache.nifi + nifi-utils + 1.7.0-SNAPSHOT + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + org.apache.nifi + nifi-ssl-context-service + + + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index efacfbbea425..e2a3ca2f0726 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -118,14 +118,12 @@ protected List getSupportedPropertyDescriptors() { } private ProxyConfigurationService proxyConfigurationService; - private SSLContextService sslContextService; private RecordReaderFactory readerFactory; private RecordPath recordPath; private OkHttpClient client; @OnEnabled public void onEnabled(final ConfigurationContext context) { - sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class); proxyConfigurationService = context.getProperty(PROXY_CONFIGURATION_SERVICE) .asControllerService(ProxyConfigurationService.class); From f8f5138dec19600cbc4b351dda485361f74357cd Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Mon, 21 May 2018 08:55:47 -0400 Subject: [PATCH 05/22] NIFI-5214 Added another tag based on code review. --- .../src/main/java/org/apache/nifi/lookup/RestLookupService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index e2a3ca2f0726..b31d6a0e7327 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -61,7 +61,7 @@ import java.util.Set; import java.util.stream.Collectors; -@Tags({ "rest", "lookup", "json", "xml" }) +@Tags({ "rest", "lookup", "json", "xml", "http" }) @CapabilityDescription("Use a REST service to enrich records.") public class RestLookupService extends AbstractControllerService implements LookupService { static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder() From f694f676bfd0e22aec15a8c8f136620f29f69172 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Mon, 21 May 2018 09:44:30 -0400 Subject: [PATCH 06/22] NIFI-5214 Added user-defined header support. --- .../apache/nifi/lookup/RestLookupService.java | 36 +++++++++++++++++ .../nifi/lookup/RestLookupServiceIT.groovy | 39 +++++++++++++++++-- .../lookup/rest/handlers/SimpleJson.groovy | 11 +++++- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index b31d6a0e7327..71e209899e4f 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -146,6 +146,21 @@ public void onEnabled(final ConfigurationContext context) { if (!StringUtils.isBlank(path)) { recordPath = RecordPath.compile(path); } + + getHeaders(context); + } + + private Map headers; + private void getHeaders(ConfigurationContext context) { + headers = new HashMap<>(); + for (PropertyDescriptor descriptor : context.getProperties().keySet()) { + if (descriptor.getName().startsWith("header.")) { + headers.put( + descriptor.getDisplayName(), + context.getProperty(descriptor).evaluateAttributeExpressions().getValue() + ); + } + } } private void setProxy(OkHttpClient.Builder builder) { @@ -192,6 +207,21 @@ public Optional lookup(Map coordinates) throws LookupFai } } + protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { + if (propertyDescriptorName.startsWith("header")) { + String header = propertyDescriptorName.substring(propertyDescriptorName.indexOf(".") + 1, propertyDescriptorName.length()); + return new PropertyDescriptor.Builder() + .name(propertyDescriptorName) + .displayName(header) + .addValidator(Validator.VALID) + .dynamic(true) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build(); + } + + return null; + } + protected Response executeRequest(Request request) throws IOException { return client.newCall(request).execute(); } @@ -257,6 +287,12 @@ private Request buildRequest(final String mimeType, final String method, final S break; } + if (headers != null) { + for (Map.Entry header : headers.entrySet()) { + request = request.addHeader(header.getKey(), header.getValue()); + } + } + return request.build(); } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index db8516a41f3a..36bab3b8fac9 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -90,10 +90,10 @@ class RestLookupServiceIT { try { server.startServer() def coordinates = [ - "schema.name": "simple", - "endpoint": server.url + "/simple_array", - "mime.type": "application/json", - "request.method": "get" + "schema.name": "simple", + "endpoint": server.url + "/simple_array", + "mime.type": "application/json", + "request.method": "get" ] Optional response = lookupService.lookup(coordinates) @@ -106,6 +106,37 @@ class RestLookupServiceIT { } } + @Test + void testHeaders() { + runner.disableControllerService(lookupService) + runner.setProperty(lookupService, "header.X-USER", "jane.doe") + runner.setProperty(lookupService, "header.X-PASS", "testing7890") + runner.enableControllerService(lookupService) + + TestServer server = new TestServer() + ServletHandler handler = new ServletHandler() + handler.addServletWithMapping(SimpleJson.class, "/simple") + server.addHandler(handler) + try { + server.startServer() + + def coordinates = [ + "schema.name": "simple", + "endpoint": server.url + "/simple", + "mime.type": "application/json", + "request.method": "get" + ] + + Optional response = lookupService.lookup(coordinates) + Assert.assertTrue(response.isPresent()) + def record = response.get() + Assert.assertEquals("jane.doe", record.getAsString("username")) + Assert.assertEquals("testing7890", record.getAsString("password")) + } finally { + server.shutdownServer() + } + } + @Test void complexJson() { runner.disableControllerService(lookupService) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy index eb17a7baa36a..bff9c74705e9 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy @@ -1,5 +1,8 @@ package org.apache.nifi.lookup.rest.handlers +import org.slf4j.Logger +import org.slf4j.LoggerFactory + import static groovy.json.JsonOutput.* import javax.servlet.http.HttpServlet @@ -7,13 +10,17 @@ import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletResponse class SimpleJson extends HttpServlet { + Logger logger = LoggerFactory.getLogger(SimpleJson.class); @Override void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + String u = request.getHeader("X-USER") + String p = request.getHeader("X-PASS") + response.contentType = "application/json" response.outputStream.write(prettyPrint( toJson([ - username: "john.smith", - password: "testing1234" + username: u ?: "john.smith", + password: p ?: "testing1234" ]) ).bytes) } From 968a0c0854d2ea3f38ef4b7979e100c14885eccf Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Mon, 21 May 2018 18:37:14 -0400 Subject: [PATCH 07/22] NIFI-5214 Added Basic Auth support. --- .../nifi-lookup-services/pom.xml | 6 ++ .../apache/nifi/lookup/RestLookupService.java | 76 ++++++++++++++++++- .../nifi/lookup/RestLookupServiceIT.groovy | 44 +++++++++++ .../lookup/rest/handlers/BasicAuth.groovy | 38 ++++++++++ 4 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml index 0fca07bc5d70..2db260656493 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml @@ -141,6 +141,12 @@ jetty-servlet test + + com.burgstaller + okhttp-digest + 1.13 + compile + diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index 71e209899e4f..a29e48310549 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -17,6 +17,10 @@ package org.apache.nifi.lookup; +import com.burgstaller.okhttp.AuthenticationCacheInterceptor; +import com.burgstaller.okhttp.CachingAuthenticatorDecorator; +import com.burgstaller.okhttp.digest.CachingAuthenticator; +import com.burgstaller.okhttp.digest.DigestAuthenticator; import okhttp3.Credentials; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -31,6 +35,8 @@ import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.proxy.ProxyConfigurationService; import org.apache.nifi.proxy.ProxySpec; @@ -59,8 +65,12 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; import java.util.stream.Collectors; +import static org.apache.commons.lang3.StringUtils.trimToEmpty; + @Tags({ "rest", "lookup", "json", "xml", "http" }) @CapabilityDescription("Use a REST service to enrich records.") public class RestLookupService extends AbstractControllerService implements LookupService { @@ -92,6 +102,31 @@ public class RestLookupService extends AbstractControllerService implements Look .required(false) .identifiesControllerService(SSLContextService.class) .build(); + public static final PropertyDescriptor PROP_BASIC_AUTH_USERNAME = new PropertyDescriptor.Builder() + .name("rest-lookup-basic-auth-username") + .displayName("Basic Authentication Username") + .description("The username to be used by the client to authenticate against the Remote URL. Cannot include control characters (0-31), ':', or DEL (127).") + .required(false) + .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$"))) + .build(); + + public static final PropertyDescriptor PROP_BASIC_AUTH_PASSWORD = new PropertyDescriptor.Builder() + .name("rest-lookup-basic-auth-password") + .displayName("Basic Authentication Password") + .description("The password to be used by the client to authenticate against the Remote URL.") + .required(false) + .sensitive(true) + .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$"))) + .build(); + public static final PropertyDescriptor PROP_DIGEST_AUTH = new PropertyDescriptor.Builder() + .name("rest-lookup-digest-auth") + .displayName("Use Digest Authentication") + .description("Whether to communicate with the website using Digest Authentication. 'Basic Authentication Username' and 'Basic Authentication Password' are used " + + "for authentication.") + .required(false) + .defaultValue("false") + .allowableValues("true", "false") + .build(); private static final ProxySpec[] PROXY_SPECS = {ProxySpec.HTTP_AUTH, ProxySpec.SOCKS}; public static final PropertyDescriptor PROXY_CONFIGURATION_SERVICE @@ -109,7 +144,10 @@ public class RestLookupService extends AbstractControllerService implements Look RECORD_READER, RECORD_PATH, SSL_CONTEXT_SERVICE, - PROXY_CONFIGURATION_SERVICE + PROXY_CONFIGURATION_SERVICE, + PROP_BASIC_AUTH_USERNAME, + PROP_BASIC_AUTH_PASSWORD, + PROP_DIGEST_AUTH )); } @@ -130,6 +168,8 @@ public void onEnabled(final ConfigurationContext context) { OkHttpClient.Builder builder = new OkHttpClient.Builder(); + setAuthenticator(builder, context); + if (proxyConfigurationService != null) { setProxy(builder); } @@ -293,9 +333,43 @@ private Request buildRequest(final String mimeType, final String method, final S } } + if (!basicUser.isEmpty() && !isDigest) { + String credential = Credentials.basic(basicUser, basicPass); + request = request.header("Authorization", credential); + } + return request.build(); } + private String basicUser; + private String basicPass; + private boolean isDigest; + + private void setAuthenticator(OkHttpClient.Builder okHttpClientBuilder, ConfigurationContext context) { + final String authUser = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_USERNAME).getValue()); + this.basicUser = authUser; + + + isDigest = context.getProperty(PROP_DIGEST_AUTH).asBoolean(); + // If the username/password properties are set then check if digest auth is being used + if (!authUser.isEmpty() && isDigest) { + final String authPass = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_PASSWORD).getValue()); + this.basicPass = authPass; + + /* + * OkHttp doesn't have built-in Digest Auth Support. A ticket for adding it is here[1] but they authors decided instead to rely on a 3rd party lib. + * + * [1] https://github.com/square/okhttp/issues/205#issuecomment-154047052 + */ + final Map authCache = new ConcurrentHashMap<>(); + com.burgstaller.okhttp.digest.Credentials credentials = new com.burgstaller.okhttp.digest.Credentials(authUser, authPass); + final DigestAuthenticator digestAuthenticator = new DigestAuthenticator(credentials); + + okHttpClientBuilder.interceptors().add(new AuthenticationCacheInterceptor(authCache)); + okHttpClientBuilder.authenticator(new CachingAuthenticatorDecorator(digestAuthenticator, authCache)); + } + } + @Override public Class getValueType() { return Record.class; diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index 36bab3b8fac9..caae68cb3edf 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -4,6 +4,7 @@ import org.apache.avro.Schema import org.apache.nifi.avro.AvroTypeUtil import org.apache.nifi.json.JsonTreeReader import org.apache.nifi.lookup.rest.SchemaUtil +import org.apache.nifi.lookup.rest.handlers.BasicAuth import org.apache.nifi.lookup.rest.handlers.ComplexJson import org.apache.nifi.lookup.rest.handlers.SimpleJson import org.apache.nifi.lookup.rest.handlers.SimpleJsonArray @@ -56,6 +57,49 @@ class RestLookupServiceIT { runner.assertValid() } + @Test + void basicAuth() { + runner.disableControllerService(lookupService) + runner.setProperty(lookupService, RestLookupService.PROP_BASIC_AUTH_USERNAME, "john.smith") + runner.setProperty(lookupService, RestLookupService.PROP_BASIC_AUTH_PASSWORD, "testing1234") + runner.enableControllerService(lookupService) + + TestServer server = new TestServer() + server.addHandler(new BasicAuth()) + try { + server.startServer() + def coordinates = [ + "schema.name": "simple", + "endpoint": server.url + "/simple", + "mime.type": "application/json", + "request.method": "get" + ] + + Optional response = lookupService.lookup(coordinates) + Assert.assertTrue(response.isPresent()) + def record = response.get() + Assert.assertEquals("john.smith", record.getAsString("username")) + Assert.assertEquals("testing1234", record.getAsString("password")) + + Throwable t + try { + runner.disableControllerService(lookupService) + runner.setProperty(lookupService, RestLookupService.PROP_BASIC_AUTH_USERNAME, "john.smith2") + runner.setProperty(lookupService, RestLookupService.PROP_BASIC_AUTH_PASSWORD, ":wetadfasdfadf") + runner.enableControllerService(lookupService) + + lookupService.lookup(coordinates) + } catch (Throwable lfe) { + t = lfe + } + + Assert.assertNotNull(t) + Assert.assertTrue(t instanceof LookupFailureException) + } finally { + server.shutdownServer() + } + } + @Test void simpleJson() { TestServer server = new TestServer() diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy new file mode 100644 index 000000000000..9ca7f1ffdeec --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy @@ -0,0 +1,38 @@ +package org.apache.nifi.lookup.rest.handlers + +import org.eclipse.jetty.server.Request +import org.eclipse.jetty.server.handler.AbstractHandler + +import javax.servlet.ServletException +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +import static groovy.json.JsonOutput.prettyPrint +import static groovy.json.JsonOutput.toJson + +class BasicAuth extends AbstractHandler { + + @Override + void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + baseRequest.handled = true + def authString = request.getHeader("Authorization") + def headers = [] + request.headerNames.each { headers << it } + + if (!authString || authString != "Basic am9obi5zbWl0aDpudWxs") { + response.status = 401 + response.setHeader("WWW-Authenticate", "Basic realm=\"Jetty\"") + response.setHeader("response.phrase", "Unauthorized") + response.contentType = "text/plain" + response.writer.println("Get off my lawn!") + return + } + + response.writer.println(prettyPrint( + toJson([ + username: "john.smith", + password: "testing1234" + ]) + )) + } +} \ No newline at end of file From a03f4096f99cd6e3c8cfcd9c463655ab01935e80 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Fri, 1 Jun 2018 13:42:13 -0400 Subject: [PATCH 08/22] NIFI-5214 Moved documentation. --- .../additionalDetails.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/{docs.org.apache.nifi.lookup.RestLookupService => docs/org.apache.nifi.lookup.RestLookupService}/additionalDetails.html (100%) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs.org.apache.nifi.lookup.RestLookupService/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html similarity index 100% rename from nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs.org.apache.nifi.lookup.RestLookupService/additionalDetails.html rename to nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html From 95bbe3f0486aefa458c8f5d48bfd326ff718c195 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Fri, 1 Jun 2018 14:58:48 -0400 Subject: [PATCH 09/22] NIFI-5214 Fixed checkstyle issues; added changes requested in a review. --- .../nifi-slack-processors/pom.xml | 3 +- .../nifi/processors/slack/PutSlackTest.java | 2 +- .../nifi-livy-processors/pom.xml | 7 ++ .../livy/TestExecuteSparkInteractive.java | 2 +- .../livy/TestExecuteSparkInteractiveSSL.java | 2 +- .../apache/nifi/lookup/RestLookupService.java | 66 ++++++++++--------- .../nifi/lookup/RestLookupServiceIT.groovy | 45 ++++++++++++- .../apache/nifi/lookup/rest/SchemaUtil.groovy | 17 +++++ .../lookup/rest/handlers/BasicAuth.groovy | 17 +++++ .../lookup/rest/handlers/ComplexJson.groovy | 17 +++++ .../nifi/lookup/rest/handlers/NoRecord.groovy | 39 +++++++++++ .../lookup/rest/handlers/SimpleJson.groovy | 17 +++++ .../rest/handlers/SimpleJsonArray.groovy | 17 +++++ 13 files changed, 214 insertions(+), 37 deletions(-) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/NoRecord.groovy diff --git a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/pom.xml b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/pom.xml index 8c2c4939f99a..a82f425e5b53 100644 --- a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/pom.xml +++ b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/pom.xml @@ -89,10 +89,9 @@ org.apache.nifi - nifi-standard-processors + nifi-standard-web-utils 1.7.0-SNAPSHOT test - test-jar diff --git a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PutSlackTest.java b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PutSlackTest.java index a6b330da179b..f6b09dca0ad6 100644 --- a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PutSlackTest.java +++ b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PutSlackTest.java @@ -19,9 +19,9 @@ import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.processor.ProcessSession; -import org.apache.nifi.processors.standard.TestServer; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; +import org.apache.nifi.web.util.TestServer; import org.eclipse.jetty.servlet.ServletHandler; import org.junit.Before; import org.junit.Test; diff --git a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/pom.xml b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/pom.xml index 56e115d3a008..354f7322f77e 100644 --- a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/pom.xml +++ b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/pom.xml @@ -89,6 +89,13 @@ org.apache.nifi nifi-kerberos-credentials-service-api + 1.7.0-SNAPSHOT + test + + + org.apache.nifi + nifi-standard-web-utils + 1.7.0-SNAPSHOT test diff --git a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/src/test/java/org/apache/nifi/processors/livy/TestExecuteSparkInteractive.java b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/src/test/java/org/apache/nifi/processors/livy/TestExecuteSparkInteractive.java index fc454ab57af2..1be718ad8177 100644 --- a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/src/test/java/org/apache/nifi/processors/livy/TestExecuteSparkInteractive.java +++ b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/src/test/java/org/apache/nifi/processors/livy/TestExecuteSparkInteractive.java @@ -17,10 +17,10 @@ package org.apache.nifi.processors.livy; import org.apache.nifi.controller.livy.LivySessionController; -import org.apache.nifi.processors.standard.TestServer; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; +import org.apache.nifi.web.util.TestServer; import org.eclipse.jetty.server.Handler; import org.junit.After; import org.junit.AfterClass; diff --git a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/src/test/java/org/apache/nifi/processors/livy/TestExecuteSparkInteractiveSSL.java b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/src/test/java/org/apache/nifi/processors/livy/TestExecuteSparkInteractiveSSL.java index 133c77314ea0..3e84cba64d0c 100644 --- a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/src/test/java/org/apache/nifi/processors/livy/TestExecuteSparkInteractiveSSL.java +++ b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/src/test/java/org/apache/nifi/processors/livy/TestExecuteSparkInteractiveSSL.java @@ -17,11 +17,11 @@ package org.apache.nifi.processors.livy; import org.apache.nifi.controller.livy.LivySessionController; -import org.apache.nifi.processors.standard.TestServer; import org.apache.nifi.ssl.StandardSSLContextService; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; +import org.apache.nifi.web.util.TestServer; import org.eclipse.jetty.server.Handler; import org.junit.After; import org.junit.AfterClass; diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index a29e48310549..685bbf4e97cc 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -35,7 +35,6 @@ import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.expression.ExpressionLanguageScope; -import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.proxy.ProxyConfigurationService; @@ -240,8 +239,10 @@ public Optional lookup(Map coordinates) throws LookupFai Record record = handleResponse(is, coordinates); - return Optional.of(record); - } catch (MalformedRecordException | SchemaNotFoundException | IOException e) { + return record != null + ? Optional.of(record) + : Optional.empty(); + } catch (Exception e) { getLogger().error("Could not execute lookup.", e); throw new LookupFailureException(e); } @@ -249,7 +250,7 @@ public Optional lookup(Map coordinates) throws LookupFai protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { if (propertyDescriptorName.startsWith("header")) { - String header = propertyDescriptorName.substring(propertyDescriptorName.indexOf(".") + 1, propertyDescriptorName.length()); + String header = propertyDescriptorName.substring(propertyDescriptorName.indexOf(".") + 1); return new PropertyDescriptor.Builder() .name(propertyDescriptorName) .displayName(header) @@ -272,36 +273,39 @@ private Record handleResponse(InputStream is, Map coordinates) t e -> e.getKey(), e -> e.getValue().toString() )); - RecordReader reader = readerFactory.createRecordReader(variables, is, getLogger()); - Record record = reader.nextRecord(); - - if (recordPath != null) { - Optional fv = recordPath.evaluate(record).getSelectedFields().findFirst(); - if (fv.isPresent()) { - FieldValue fieldValue = fv.get(); - RecordSchema schema = new SimpleRecordSchema(Arrays.asList(fieldValue.getField())); - String[] parts = recordPath.getPath().split("/"); - String last = parts[parts.length - 1]; - - Record temp; - Object value = fieldValue.getValue(); - if (value instanceof Record) { - temp = (Record)value; - } else if (value instanceof Map) { - temp = new MapRecord(schema, (Map)value); - } else { - temp = new MapRecord(schema, new HashMap(){{ - put(last, value); - }}); + try (RecordReader reader = readerFactory.createRecordReader(variables, is, getLogger())) { + + Record record = reader.nextRecord(); + + if (recordPath != null) { + Optional fv = recordPath.evaluate(record).getSelectedFields().findFirst(); + if (fv.isPresent()) { + FieldValue fieldValue = fv.get(); + RecordSchema schema = new SimpleRecordSchema(Arrays.asList(fieldValue.getField())); + String[] parts = recordPath.getPath().split("/"); + String last = parts[parts.length - 1]; + + Record temp; + Object value = fieldValue.getValue(); + if (value instanceof Record) { + temp = (Record) value; + } else if (value instanceof Map) { + temp = new MapRecord(schema, (Map) value); + } else { + temp = new MapRecord(schema, new HashMap() {{ + put(last, value); + }}); + } + + record = temp; } - - record = temp; } - } - reader.close(); - - return record; + return record; + } catch (Exception ex) { + is.close(); + throw new RuntimeException(ex); + } } private Request buildRequest(final String mimeType, final String method, final String body, final String endpoint) { diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index caae68cb3edf..2656eef535e3 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.nifi.lookup import org.apache.avro.Schema @@ -6,6 +23,7 @@ import org.apache.nifi.json.JsonTreeReader import org.apache.nifi.lookup.rest.SchemaUtil import org.apache.nifi.lookup.rest.handlers.BasicAuth import org.apache.nifi.lookup.rest.handlers.ComplexJson +import org.apache.nifi.lookup.rest.handlers.NoRecord import org.apache.nifi.lookup.rest.handlers.SimpleJson import org.apache.nifi.lookup.rest.handlers.SimpleJsonArray import org.apache.nifi.schema.access.SchemaAccessUtils @@ -94,7 +112,7 @@ class RestLookupServiceIT { } Assert.assertNotNull(t) - Assert.assertTrue(t instanceof LookupFailureException) + Assert.assertTrue(t.getClass().getCanonicalName(), t instanceof LookupFailureException) } finally { server.shutdownServer() } @@ -125,6 +143,31 @@ class RestLookupServiceIT { } } + @Test + void noRecord() { + TestServer server = new TestServer() + ServletHandler handler = new ServletHandler() + handler.addServletWithMapping(NoRecord.class, "/simple") + server.addHandler(handler) + try { + server.startServer() + def coordinates = [ + "schema.name": "simple", + "endpoint": server.url + "/simple", + "mime.type": "application/json", + "request.method": "get" + ] + + Optional response = lookupService.lookup(coordinates) + Assert.assertTrue(response.isPresent()) + def record = response.get() + Assert.assertNull(record.getAsString("username")) + Assert.assertNull(record.getAsString("password")) + } finally { + server.shutdownServer() + } + } + @Test void simpleJsonArray() { TestServer server = new TestServer() diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy index 49ece37db2dc..a152f90d48a7 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.nifi.lookup.rest class SchemaUtil { diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy index 9ca7f1ffdeec..c0c833e67263 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.nifi.lookup.rest.handlers import org.eclipse.jetty.server.Request diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy index 1e714781bef1..299132297fbb 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.nifi.lookup.rest.handlers import javax.servlet.http.HttpServlet diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/NoRecord.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/NoRecord.groovy new file mode 100644 index 000000000000..68933db58bf6 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/NoRecord.groovy @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.lookup.rest.handlers + +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import static groovy.json.JsonOutput.* + +import javax.servlet.http.HttpServlet +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +class NoRecord extends HttpServlet { + Logger logger = LoggerFactory.getLogger(SimpleJson.class); + @Override + void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + + response.contentType = "application/json" + response.outputStream.write(prettyPrint( + toJson([:]) + ).bytes) + } +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy index bff9c74705e9..b4115857407c 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.nifi.lookup.rest.handlers import org.slf4j.Logger diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJsonArray.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJsonArray.groovy index 084525999802..6b63dd68795e 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJsonArray.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJsonArray.groovy @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.nifi.lookup.rest.handlers import javax.servlet.http.HttpServlet From a7c6e22b64255657a0e70316cb83102091eb5330 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Sat, 2 Jun 2018 06:51:06 -0400 Subject: [PATCH 10/22] NIFI-5214 Added changes requested in a code review. --- .../apache/nifi/lookup/RestLookupService.java | 76 +++++++++++-------- .../additionalDetails.html | 4 + .../nifi/lookup/RestLookupServiceIT.groovy | 4 +- .../nifi/lookup/TestRestLookupService.groovy | 1 + 4 files changed, 50 insertions(+), 35 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index 685bbf4e97cc..727365adbe10 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -41,6 +41,7 @@ import org.apache.nifi.proxy.ProxySpec; import org.apache.nifi.record.path.FieldValue; import org.apache.nifi.record.path.RecordPath; +import org.apache.nifi.record.path.validation.RecordPathValidator; import org.apache.nifi.schema.access.SchemaNotFoundException; import org.apache.nifi.serialization.MalformedRecordException; import org.apache.nifi.serialization.RecordReader; @@ -79,7 +80,6 @@ public class RestLookupService extends AbstractControllerService implements Look .description("The record reader to use for loading the payload and handling it as a record set.") .expressionLanguageSupported(ExpressionLanguageScope.NONE) .identifiesControllerService(RecordReaderFactory.class) - .addValidator(Validator.VALID) .required(true) .build(); @@ -89,6 +89,16 @@ public class RestLookupService extends AbstractControllerService implements Look .description("An optional record path that can be used to define where in a record to get the real data to merge " + "into the record set to be enriched. See documentation for examples of when this might be useful.") .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .addValidator(new RecordPathValidator()) + .required(false) + .build(); + + static final PropertyDescriptor RECORD_PATH_PROPERTY_NAME = new PropertyDescriptor.Builder() + .name("rest-lookup-record-path-name") + .displayName("Record Path Property Name") + .description("An optional name for the property loaded by the record path. This will be the key used for the value " + + "when the loaded record is merged into record to be enriched. See docs for more details.") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) .addValidator(Validator.VALID) .required(false) .build(); @@ -137,17 +147,24 @@ public class RestLookupService extends AbstractControllerService implements Look static final String METHOD_KEY = "request.method"; static final List DESCRIPTORS; + static final Set KEYS; static { DESCRIPTORS = Collections.unmodifiableList(Arrays.asList( RECORD_READER, RECORD_PATH, + RECORD_PATH_PROPERTY_NAME, SSL_CONTEXT_SERVICE, PROXY_CONFIGURATION_SERVICE, PROP_BASIC_AUTH_USERNAME, PROP_BASIC_AUTH_PASSWORD, PROP_DIGEST_AUTH )); + Set _keys = new HashSet<>(); + _keys.add(ENDPOINT_KEY); + _keys.add(MIME_TYPE_KEY); + _keys.add(METHOD_KEY); + KEYS = Collections.unmodifiableSet(_keys); } protected List getSupportedPropertyDescriptors() { @@ -158,6 +175,11 @@ protected List getSupportedPropertyDescriptors() { private RecordReaderFactory readerFactory; private RecordPath recordPath; private OkHttpClient client; + private Map headers; + private volatile String recordPathName; + private volatile String basicUser; + private volatile String basicPass; + private volatile boolean isDigest; @OnEnabled public void onEnabled(final ConfigurationContext context) { @@ -186,14 +208,17 @@ public void onEnabled(final ConfigurationContext context) { recordPath = RecordPath.compile(path); } + recordPathName = context.getProperty(RECORD_PATH_PROPERTY_NAME).isSet() + ? context.getProperty(RECORD_PATH_PROPERTY_NAME).evaluateAttributeExpressions().getValue() + : null; + getHeaders(context); } - private Map headers; private void getHeaders(ConfigurationContext context) { headers = new HashMap<>(); for (PropertyDescriptor descriptor : context.getProperties().keySet()) { - if (descriptor.getName().startsWith("header.")) { + if (descriptor.isDynamic()) { headers.put( descriptor.getDisplayName(), context.getProperty(descriptor).evaluateAttributeExpressions().getValue() @@ -239,9 +264,7 @@ public Optional lookup(Map coordinates) throws LookupFai Record record = handleResponse(is, coordinates); - return record != null - ? Optional.of(record) - : Optional.empty(); + return Optional.ofNullable(record); } catch (Exception e) { getLogger().error("Could not execute lookup.", e); throw new LookupFailureException(e); @@ -249,18 +272,13 @@ public Optional lookup(Map coordinates) throws LookupFai } protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { - if (propertyDescriptorName.startsWith("header")) { - String header = propertyDescriptorName.substring(propertyDescriptorName.indexOf(".") + 1); - return new PropertyDescriptor.Builder() - .name(propertyDescriptorName) - .displayName(header) - .addValidator(Validator.VALID) - .dynamic(true) - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .build(); - } - - return null; + return new PropertyDescriptor.Builder() + .name(propertyDescriptorName) + .displayName(propertyDescriptorName) + .addValidator(Validator.VALID) + .dynamic(true) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .build(); } protected Response executeRequest(Request request) throws IOException { @@ -282,8 +300,6 @@ private Record handleResponse(InputStream is, Map coordinates) t if (fv.isPresent()) { FieldValue fieldValue = fv.get(); RecordSchema schema = new SimpleRecordSchema(Arrays.asList(fieldValue.getField())); - String[] parts = recordPath.getPath().split("/"); - String last = parts[parts.length - 1]; Record temp; Object value = fieldValue.getValue(); @@ -292,19 +308,21 @@ private Record handleResponse(InputStream is, Map coordinates) t } else if (value instanceof Map) { temp = new MapRecord(schema, (Map) value); } else { - temp = new MapRecord(schema, new HashMap() {{ - put(last, value); - }}); + Map val = new HashMap<>(); + val.put(recordPathName, value); + temp = new MapRecord(schema, val); } record = temp; + } else { + record = null; } } return record; } catch (Exception ex) { is.close(); - throw new RuntimeException(ex); + throw ex; } } @@ -345,10 +363,6 @@ private Request buildRequest(final String mimeType, final String method, final S return request.build(); } - private String basicUser; - private String basicPass; - private boolean isDigest; - private void setAuthenticator(OkHttpClient.Builder okHttpClientBuilder, ConfigurationContext context) { final String authUser = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_USERNAME).getValue()); this.basicUser = authUser; @@ -381,10 +395,6 @@ public Class getValueType() { @Override public Set getRequiredKeys() { - return new HashSet(){{ - add(ENDPOINT_KEY); - add(MIME_TYPE_KEY); - add(METHOD_KEY); - }}; + return KEYS; } } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html index 1bba4e733f7e..34e9240e0527 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html @@ -20,6 +20,7 @@ +

General

This lookup service has the following required keys:

  • endpoint
    • @@ -38,5 +39,8 @@

    The record reader is used to consume the response of the REST service call and turn it into one or more records. The record path property is provided to allow for a lookup path to either a nested record or a single point deep in the REST response. Note: a valid schema must be built that encapsulates the REST response accurately in order for this service to work.

    +

    Headers

    +

    Headers are supported using dynamic properties. Just add a dynamic property and the name will be the header name and the value will be the value for the header. Expression language + powered by input from the variable registry is supported.

    \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index 2656eef535e3..2eaf87fe00be 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -196,8 +196,8 @@ class RestLookupServiceIT { @Test void testHeaders() { runner.disableControllerService(lookupService) - runner.setProperty(lookupService, "header.X-USER", "jane.doe") - runner.setProperty(lookupService, "header.X-PASS", "testing7890") + runner.setProperty(lookupService, "X-USER", "jane.doe") + runner.setProperty(lookupService, "X-PASS", "testing7890") runner.enableControllerService(lookupService) TestServer server = new TestServer() diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy index aac1f9671209..1c7c81ea8842 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy @@ -111,6 +111,7 @@ class TestRestLookupService { runner.disableControllerService(lookupService) runner.setProperty(lookupService, RestLookupService.RECORD_PATH, "/person/sport") + runner.setProperty(lookupService, RestLookupService.RECORD_PATH_PROPERTY_NAME, "sport") runner.enableControllerService(lookupService) runner.assertValid() From b78d00a75af1eaf713f3ef7935cc5c50cde7b362 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Sat, 2 Jun 2018 09:26:19 -0400 Subject: [PATCH 11/22] NIFI-5214 Added verb test and @DynamicProperties --- .../apache/nifi/lookup/RestLookupService.java | 6 ++ .../nifi/lookup/RestLookupServiceIT.groovy | 49 +++++++++++++ .../nifi/lookup/rest/handlers/NoRecord.groovy | 2 +- .../lookup/rest/handlers/SimpleJson.groovy | 2 +- .../nifi/lookup/rest/handlers/VerbTest.groovy | 69 +++++++++++++++++++ 5 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/VerbTest.groovy diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index 727365adbe10..7596f14f7a67 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -27,6 +27,8 @@ import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; +import org.apache.nifi.annotation.behavior.DynamicProperties; +import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnEnabled; @@ -73,6 +75,10 @@ @Tags({ "rest", "lookup", "json", "xml", "http" }) @CapabilityDescription("Use a REST service to enrich records.") +@DynamicProperties({ + @DynamicProperty(name = "*", value = "*", description = "All dynamic properties are added as HTTP headers with the name " + + "as the header name and the value as the header value.") +}) public class RestLookupService extends AbstractControllerService implements LookupService { static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder() .name("rest-lookup-record-reader") diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index 2eaf87fe00be..b86b61f30e23 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -26,6 +26,7 @@ import org.apache.nifi.lookup.rest.handlers.ComplexJson import org.apache.nifi.lookup.rest.handlers.NoRecord import org.apache.nifi.lookup.rest.handlers.SimpleJson import org.apache.nifi.lookup.rest.handlers.SimpleJsonArray +import org.apache.nifi.lookup.rest.handlers.VerbTest import org.apache.nifi.schema.access.SchemaAccessUtils import org.apache.nifi.serialization.record.MockSchemaRegistry import org.apache.nifi.serialization.record.Record @@ -38,6 +39,9 @@ import org.junit.Assert import org.junit.Before import org.junit.Test +import static groovy.json.JsonOutput.prettyPrint +import static groovy.json.JsonOutput.toJson + class RestLookupServiceIT { static final JsonTreeReader reader static final MockSchemaRegistry registry = new MockSchemaRegistry() @@ -253,4 +257,49 @@ class RestLookupServiceIT { server.shutdownServer() } } + + @Test + void testOtherVerbs() { + TestServer server = new TestServer() + ServletHandler handler = new ServletHandler() + handler.addServletWithMapping(VerbTest.class, "/simple") + server.addHandler(handler) + try { + runner.disableControllerService(lookupService) + runner.setProperty(lookupService, "needs-body", "true") + runner.enableControllerService(lookupService) + server.startServer() + + def validation = { String verb, boolean addBody -> + def coordinates = [ + "schema.name" : "simple", + "endpoint" : server.url + "/simple", + "mime.type" : "application/json", + "request.method": verb + ] + + if (addBody) { + coordinates["request.body"] = prettyPrint(toJson([ msg: "Hello, world" ])) + } + + Optional response = lookupService.lookup(coordinates) + Assert.assertTrue(response.isPresent()) + def record = response.get() + Assert.assertEquals("john.smith", record.getAsString("username")) + Assert.assertEquals("testing1234", record.getAsString("password")) + } + + ["delete", "post", "put"].each { verb -> + validation(verb, true) + } + + runner.disableControllerService(lookupService) + runner.setProperty(lookupService, "needs-body", "") + runner.enableControllerService(lookupService) + + validation("delete", false) + } finally { + server.shutdownServer() + } + } } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/NoRecord.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/NoRecord.groovy index 68933db58bf6..cf80b640d0bb 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/NoRecord.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/NoRecord.groovy @@ -27,7 +27,7 @@ import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletResponse class NoRecord extends HttpServlet { - Logger logger = LoggerFactory.getLogger(SimpleJson.class); + Logger logger = LoggerFactory.getLogger(NoRecord.class) @Override void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy index b4115857407c..9fcc9d2f04b4 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/SimpleJson.groovy @@ -27,7 +27,7 @@ import javax.servlet.http.HttpServletRequest import javax.servlet.http.HttpServletResponse class SimpleJson extends HttpServlet { - Logger logger = LoggerFactory.getLogger(SimpleJson.class); + Logger logger = LoggerFactory.getLogger(SimpleJson.class) @Override void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { String u = request.getHeader("X-USER") diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/VerbTest.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/VerbTest.groovy new file mode 100644 index 000000000000..ad6f232f3bf1 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/VerbTest.groovy @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.lookup.rest.handlers + +import org.apache.nifi.util.StringUtils +import org.junit.Assert +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import javax.servlet.http.HttpServlet +import javax.servlet.http.HttpServletRequest +import javax.servlet.http.HttpServletResponse + +import static groovy.json.JsonOutput.prettyPrint +import static groovy.json.JsonOutput.toJson + +class VerbTest extends HttpServlet { + Logger logger = LoggerFactory.getLogger(VerbTest.class) + + void doDelete(HttpServletRequest request, HttpServletResponse response) { + validateBody(request) + sendResponse(response) + } + + void doPost(HttpServletRequest request, HttpServletResponse response) { + validateBody(request) + sendResponse(response) + } + + void doPut(HttpServletRequest request, HttpServletResponse response) { + validateBody(request) + sendResponse(response) + } + + void sendResponse(HttpServletResponse response) { + response.contentType = "application/json" + response.outputStream.write(prettyPrint( + toJson([ + username: "john.smith", + password: "testing1234" + ]) + ).bytes) + } + + void validateBody(HttpServletRequest request) { + String needsBody = request.getHeader("needs-body") + boolean bodyRequired = !StringUtils.isBlank(needsBody) + String body = request.inputStream.text + if (bodyRequired) { + Assert.assertNotNull(body) + Assert.assertFalse(StringUtils.isBlank(body)) + } + } +} From 6c18a5956824e048a42ef2edb03ec560bc273f93 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Mon, 4 Jun 2018 08:16:35 -0400 Subject: [PATCH 12/22] NIFI-5214 Added templated URL support to RestLookupService. --- .../nifi-lookup-services/pom.xml | 5 ++++ .../apache/nifi/lookup/RestLookupService.java | 19 ++++++++++++- .../nifi/lookup/RestLookupServiceIT.groovy | 28 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml index 2db260656493..635bc3b0821b 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml @@ -41,6 +41,11 @@ org.apache.nifi nifi-record + + org.apache.nifi + nifi-expression-language + 1.7.0-SNAPSHOT + org.apache.commons commons-configuration2 diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index 7596f14f7a67..ea6c96d97a2d 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -32,12 +32,14 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.attribute.expression.language.PreparedQuery; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.Validator; import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.attribute.expression.language.Query; import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.proxy.ProxyConfigurationService; import org.apache.nifi.proxy.ProxySpec; @@ -148,6 +150,7 @@ public class RestLookupService extends AbstractControllerService implements Look = ProxyConfiguration.createProxyConfigPropertyDescriptor(true, PROXY_SPECS); static final String ENDPOINT_KEY = "endpoint"; + static final String ENDPOINT_TEMPLATE_KEY = "endpoint.template"; static final String MIME_TYPE_KEY = "mime.type"; static final String BODY_KEY = "request.body"; static final String METHOD_KEY = "request.method"; @@ -252,7 +255,7 @@ private void setProxy(OkHttpClient.Builder builder) { @Override public Optional lookup(Map coordinates) throws LookupFailureException { - final String endpoint = (String)coordinates.get(ENDPOINT_KEY); + final String endpoint = determineEndpoint(coordinates); final String mimeType = (String)coordinates.get(MIME_TYPE_KEY); final String method = (String)coordinates.get(METHOD_KEY); final String body = (String)coordinates.get(BODY_KEY); @@ -277,6 +280,20 @@ public Optional lookup(Map coordinates) throws LookupFai } } + protected String determineEndpoint(Map coordinates) { + if (coordinates.containsKey(ENDPOINT_KEY) && coordinates.containsKey(ENDPOINT_TEMPLATE_KEY)) { + Map converted = coordinates.entrySet().stream() + .collect(Collectors.toMap( + e -> e.getKey(), + e -> e.getValue().toString() + )); + final PreparedQuery query = Query.prepare((String)coordinates.get(ENDPOINT_KEY)); + return query.evaluateExpressions(converted, null); + } else { + return (String)coordinates.get(ENDPOINT_KEY); + } + } + protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { return new PropertyDescriptor.Builder() .name(propertyDescriptorName) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index b86b61f30e23..c8d9efcf7d8a 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -302,4 +302,32 @@ class RestLookupServiceIT { server.shutdownServer() } } + + @Test + void testTemplateMode() { + TestServer server = new TestServer() + ServletHandler handler = new ServletHandler() + handler.addServletWithMapping(SimpleJson.class, "/simple/john.smith/friends/12345") + server.addHandler(handler) + try { + server.startServer() + def coordinates = [ + "schema.name": "simple", + "endpoint": server.url + '/simple/${user.name}/friends/${friend.id}', + "mime.type": "application/json", + "request.method": "get", + "user.name": "john.smith", + "friend.id": 12345, + "endpoint.template": true + ] + + Optional response = lookupService.lookup(coordinates) + Assert.assertTrue(response.isPresent()) + def record = response.get() + Assert.assertEquals("john.smith", record.getAsString("username")) + Assert.assertEquals("testing1234", record.getAsString("password")) + } finally { + server.shutdownServer() + } + } } From d7b09ea324403ad816f235eda72537794f55e323 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Mon, 4 Jun 2018 08:25:14 -0400 Subject: [PATCH 13/22] NIFI-5214 Fixed documentation based on code review changes. --- .../additionalDetails.html | 9 +++++++++ .../org/apache/nifi/lookup/RestLookupServiceIT.groovy | 2 +- .../apache/nifi/lookup/rest/handlers/ComplexJson.groovy | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html index 34e9240e0527..e67d9f3023f2 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html @@ -42,5 +42,14 @@

    General

    Headers

    Headers are supported using dynamic properties. Just add a dynamic property and the name will be the header name and the value will be the value for the header. Expression language powered by input from the variable registry is supported.

    +

    Dynamic URLs

    +

    The endpoint lookup key can be dynamically configured with Expression Language if the optional key endpoint.template is added to the + list of lookup keys. Any non-empty value for that key will work, including hard-coded values.

    +

    Ex. URL: http://example.com/service/${user.name}/friend/${friend.id}, combined with example record paths:

    +
      +
    • user.name => "/example/username"
      • +
    • friend.id => "/example/first_friend"
      • +
    +

    Would dynamically produce an endpoint of http://example.com/service/john.smith/friend/12345

    \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index c8d9efcf7d8a..94c751b89557 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -252,7 +252,7 @@ class RestLookupServiceIT { def record = response.get() Assert.assertEquals("jane.doe", record.getAsString("username")) Assert.assertEquals("testing7890", record.getAsString("password")) - Assert.assertEquals("jane.doe@company.com", record.getAsString("email")) + Assert.assertEquals("jane.doe@test-example.com", record.getAsString("email")) } finally { server.shutdownServer() } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy index 299132297fbb..c8790f89fa83 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/ComplexJson.groovy @@ -35,7 +35,7 @@ class ComplexJson extends HttpServlet { inner: [ "username": "jane.doe", "password": "testing7890", - "email": "jane.doe@company.com" + "email": "jane.doe@test-example.com" ] ] ] From c2c606ce652eb586c0fe54594ac52f8b51b087f6 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Tue, 5 Jun 2018 07:50:44 -0400 Subject: [PATCH 14/22] NIFI-5214 Changed RestLookupService to use a property descriptor. --- .../apache/nifi/lookup/RestLookupService.java | 42 ++++++++++++------- .../nifi/lookup/RestLookupServiceIT.groovy | 36 ++++++++++++---- .../nifi/lookup/TestRestLookupService.groovy | 10 ++--- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index ea6c96d97a2d..80ff4cdbd0b0 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -31,15 +31,16 @@ import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnDisabled; import org.apache.nifi.annotation.lifecycle.OnEnabled; import org.apache.nifi.attribute.expression.language.PreparedQuery; +import org.apache.nifi.attribute.expression.language.Query; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.Validator; import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.processor.util.StandardValidators; -import org.apache.nifi.attribute.expression.language.Query; import org.apache.nifi.proxy.ProxyConfiguration; import org.apache.nifi.proxy.ProxyConfigurationService; import org.apache.nifi.proxy.ProxySpec; @@ -82,6 +83,16 @@ "as the header name and the value as the header value.") }) public class RestLookupService extends AbstractControllerService implements LookupService { + static final PropertyDescriptor URL = new PropertyDescriptor.Builder() + .name("rest-lookup-url") + .displayName("URL") + .description("The URL for the REST endpoint. Expression language is evaluated against the lookup key/value pairs, " + + "not flowfile attributes or variable registry.") + .expressionLanguageSupported(ExpressionLanguageScope.NONE) + .required(true) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder() .name("rest-lookup-record-reader") .displayName("Record Reader") @@ -149,8 +160,6 @@ public class RestLookupService extends AbstractControllerService implements Look public static final PropertyDescriptor PROXY_CONFIGURATION_SERVICE = ProxyConfiguration.createProxyConfigPropertyDescriptor(true, PROXY_SPECS); - static final String ENDPOINT_KEY = "endpoint"; - static final String ENDPOINT_TEMPLATE_KEY = "endpoint.template"; static final String MIME_TYPE_KEY = "mime.type"; static final String BODY_KEY = "request.body"; static final String METHOD_KEY = "request.method"; @@ -160,6 +169,7 @@ public class RestLookupService extends AbstractControllerService implements Look static { DESCRIPTORS = Collections.unmodifiableList(Arrays.asList( + URL, RECORD_READER, RECORD_PATH, RECORD_PATH_PROPERTY_NAME, @@ -170,7 +180,6 @@ public class RestLookupService extends AbstractControllerService implements Look PROP_DIGEST_AUTH )); Set _keys = new HashSet<>(); - _keys.add(ENDPOINT_KEY); _keys.add(MIME_TYPE_KEY); _keys.add(METHOD_KEY); KEYS = Collections.unmodifiableSet(_keys); @@ -185,6 +194,7 @@ protected List getSupportedPropertyDescriptors() { private RecordPath recordPath; private OkHttpClient client; private Map headers; + private volatile PreparedQuery compiledQuery; private volatile String recordPathName; private volatile String basicUser; private volatile String basicPass; @@ -222,6 +232,13 @@ public void onEnabled(final ConfigurationContext context) { : null; getHeaders(context); + + compiledQuery = Query.prepare(context.getProperty(URL).getValue()); + } + + @OnDisabled + public void onDisabled() { + this.compiledQuery = null; } private void getHeaders(ConfigurationContext context) { @@ -281,17 +298,12 @@ public Optional lookup(Map coordinates) throws LookupFai } protected String determineEndpoint(Map coordinates) { - if (coordinates.containsKey(ENDPOINT_KEY) && coordinates.containsKey(ENDPOINT_TEMPLATE_KEY)) { - Map converted = coordinates.entrySet().stream() - .collect(Collectors.toMap( - e -> e.getKey(), - e -> e.getValue().toString() - )); - final PreparedQuery query = Query.prepare((String)coordinates.get(ENDPOINT_KEY)); - return query.evaluateExpressions(converted, null); - } else { - return (String)coordinates.get(ENDPOINT_KEY); - } + Map converted = coordinates.entrySet().stream() + .collect(Collectors.toMap( + e -> e.getKey(), + e -> e.getValue().toString() + )); + return compiledQuery.evaluateExpressions(converted, null); } protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index 94c751b89557..802097e16f00 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -90,9 +90,11 @@ class RestLookupServiceIT { server.addHandler(new BasicAuth()) try { server.startServer() + + setEndpoint(server.url, "/simple") + def coordinates = [ "schema.name": "simple", - "endpoint": server.url + "/simple", "mime.type": "application/json", "request.method": "get" ] @@ -130,9 +132,11 @@ class RestLookupServiceIT { server.addHandler(handler) try { server.startServer() + + setEndpoint(server.url, "/simple") + def coordinates = [ "schema.name": "simple", - "endpoint": server.url + "/simple", "mime.type": "application/json", "request.method": "get" ] @@ -155,9 +159,11 @@ class RestLookupServiceIT { server.addHandler(handler) try { server.startServer() + + setEndpoint(server.url, "/simple") + def coordinates = [ "schema.name": "simple", - "endpoint": server.url + "/simple", "mime.type": "application/json", "request.method": "get" ] @@ -180,9 +186,11 @@ class RestLookupServiceIT { server.addHandler(handler) try { server.startServer() + + setEndpoint(server.url, "/simple_array") + def coordinates = [ "schema.name": "simple", - "endpoint": server.url + "/simple_array", "mime.type": "application/json", "request.method": "get" ] @@ -211,9 +219,10 @@ class RestLookupServiceIT { try { server.startServer() + setEndpoint(server.url, "/simple") + def coordinates = [ "schema.name": "simple", - "endpoint": server.url + "/simple", "mime.type": "application/json", "request.method": "get" ] @@ -240,9 +249,11 @@ class RestLookupServiceIT { server.addHandler(handler) try { server.startServer() + + setEndpoint(server.url, "/complex") + def coordinates = [ "schema.name": "complex", - "endpoint": server.url + "/complex", "mime.type": "application/json", "request.method": "get" ] @@ -270,10 +281,11 @@ class RestLookupServiceIT { runner.enableControllerService(lookupService) server.startServer() + setEndpoint(server.url, "/simple") + def validation = { String verb, boolean addBody -> def coordinates = [ "schema.name" : "simple", - "endpoint" : server.url + "/simple", "mime.type" : "application/json", "request.method": verb ] @@ -311,9 +323,11 @@ class RestLookupServiceIT { server.addHandler(handler) try { server.startServer() + + setEndpoint(server.url, '/simple/${user.name}/friends/${friend.id}') + def coordinates = [ "schema.name": "simple", - "endpoint": server.url + '/simple/${user.name}/friends/${friend.id}', "mime.type": "application/json", "request.method": "get", "user.name": "john.smith", @@ -330,4 +344,10 @@ class RestLookupServiceIT { server.shutdownServer() } } + + void setEndpoint(String serverUrl, String endpoint) { + runner.disableControllerService(lookupService) + runner.setProperty(lookupService, RestLookupService.URL, serverUrl + endpoint) + runner.enableControllerService(lookupService) + } } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy index 1c7c81ea8842..5112ac46b8c5 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy @@ -49,6 +49,7 @@ class TestRestLookupService { runner.addControllerService("recordReader", recordReader) runner.setProperty(lookupService, RestLookupService.RECORD_READER, "recordReader") runner.setProperty("Lookup Service", "lookupService") + runner.setProperty(lookupService, RestLookupService.URL, "http://localhost:8080") runner.enableControllerService(lookupService) runner.enableControllerService(recordReader) runner.assertValid() @@ -65,7 +66,7 @@ class TestRestLookupService { recordReader.addRecord("Sally Doe", 47, "Curling") lookupService.response = buildResponse(toJson([ simpleTest: true]), JSON_TYPE) - def result = lookupService.lookup(getCoordinates("http://localhost:8080", JSON_TYPE, "get")) + def result = lookupService.lookup(getCoordinates(JSON_TYPE, "get")) Assert.assertTrue(result.isPresent()) def record = result.get() Assert.assertEquals("John Doe", record.getAsString("name")) @@ -97,7 +98,7 @@ class TestRestLookupService { }})) lookupService.response = buildResponse(toJson([ simpleTest: true]), JSON_TYPE) - def result = lookupService.lookup(getCoordinates("http://localhost:8080", JSON_TYPE, "get")) + def result = lookupService.lookup(getCoordinates(JSON_TYPE, "get")) Assert.assertTrue(result.isPresent()) def record = result.get() @@ -115,16 +116,15 @@ class TestRestLookupService { runner.enableControllerService(lookupService) runner.assertValid() - result = lookupService.lookup(getCoordinates("http://localhost:8080", JSON_TYPE, "get")) + result = lookupService.lookup(getCoordinates(JSON_TYPE, "get")) Assert.assertTrue(result.isPresent()) record = result.get() Assert.assertNotNull(record.getAsString("sport")) Assert.assertEquals("Soccer", record.getAsString("sport")) } - private Map getCoordinates(String endpoint, String mimeType, String method) { + private Map getCoordinates(String mimeType, String method) { def retVal = [:] - retVal[RestLookupService.ENDPOINT_KEY] = endpoint retVal[RestLookupService.MIME_TYPE_KEY] = mimeType retVal[RestLookupService.METHOD_KEY] = method From 155f31c57540a378b0f176d314e9b644ea4fc9b4 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Tue, 5 Jun 2018 07:53:35 -0400 Subject: [PATCH 15/22] NIFI-5214 Updated documentation. --- .../additionalDetails.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html index e67d9f3023f2..62ab95a250f0 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html @@ -23,7 +23,6 @@

    General

    This lookup service has the following required keys:

      -
    • endpoint
    • mime.type
    • request.method; valid values:
        @@ -43,8 +42,8 @@

        Headers

        Headers are supported using dynamic properties. Just add a dynamic property and the name will be the header name and the value will be the value for the header. Expression language powered by input from the variable registry is supported.

        Dynamic URLs

        -

        The endpoint lookup key can be dynamically configured with Expression Language if the optional key endpoint.template is added to the - list of lookup keys. Any non-empty value for that key will work, including hard-coded values.

        +

        The URL property supports expression language in a non-standard way: through the lookup key/value pairs configured on the processor. The configuration specified by the user will be passed + through to the expression language engine for evaluation. Note: flowfile attributes and variable registry settings will be disregarded here for this property.

        Ex. URL: http://example.com/service/${user.name}/friend/${friend.id}, combined with example record paths:

        • user.name => "/example/username"
          • From 51953dff9d157832a6ec47edaa62440b33f546d7 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Fri, 8 Jun 2018 08:28:47 -0400 Subject: [PATCH 16/22] NIFI-5214 Made changes requested in code review. --- .../apache/nifi/lookup/RestLookupService.java | 29 ++++++++++++------- .../additionalDetails.html | 2 +- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index 80ff4cdbd0b0..08172cdb833a 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -65,7 +65,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -82,7 +81,7 @@ @DynamicProperty(name = "*", value = "*", description = "All dynamic properties are added as HTTP headers with the name " + "as the header name and the value as the header value.") }) -public class RestLookupService extends AbstractControllerService implements LookupService { +public class RestLookupService extends AbstractControllerService implements RecordLookupService { static final PropertyDescriptor URL = new PropertyDescriptor.Builder() .name("rest-lookup-url") .displayName("URL") @@ -135,6 +134,7 @@ public class RestLookupService extends AbstractControllerService implements Look .displayName("Basic Authentication Username") .description("The username to be used by the client to authenticate against the Remote URL. Cannot include control characters (0-31), ':', or DEL (127).") .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$"))) .build(); @@ -144,6 +144,7 @@ public class RestLookupService extends AbstractControllerService implements Look .description("The password to be used by the client to authenticate against the Remote URL.") .required(false) .sensitive(true) + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$"))) .build(); public static final PropertyDescriptor PROP_DIGEST_AUTH = new PropertyDescriptor.Builder() @@ -167,6 +168,8 @@ public class RestLookupService extends AbstractControllerService implements Look static final List DESCRIPTORS; static final Set KEYS; + static final List VALID_VERBS = Arrays.asList("delete", "get", "post", "put"); + static { DESCRIPTORS = Collections.unmodifiableList(Arrays.asList( URL, @@ -179,10 +182,7 @@ public class RestLookupService extends AbstractControllerService implements Look PROP_BASIC_AUTH_PASSWORD, PROP_DIGEST_AUTH )); - Set _keys = new HashSet<>(); - _keys.add(MIME_TYPE_KEY); - _keys.add(METHOD_KEY); - KEYS = Collections.unmodifiableSet(_keys); + KEYS = Collections.emptySet(); } protected List getSupportedPropertyDescriptors() { @@ -274,9 +274,11 @@ private void setProxy(OkHttpClient.Builder builder) { public Optional lookup(Map coordinates) throws LookupFailureException { final String endpoint = determineEndpoint(coordinates); final String mimeType = (String)coordinates.get(MIME_TYPE_KEY); - final String method = (String)coordinates.get(METHOD_KEY); + final String method = ((String)coordinates.getOrDefault(METHOD_KEY, "get")).trim(); final String body = (String)coordinates.get(BODY_KEY); + validateVerb(method); + if (StringUtils.isBlank(body) && (method.equals("post") || method.equals("put"))) { throw new LookupFailureException( String.format("Used HTTP verb %s without specifying the %s key to provide a payload.", method, BODY_KEY) @@ -297,6 +299,13 @@ public Optional lookup(Map coordinates) throws LookupFai } } + protected void validateVerb(String endpoint) throws LookupFailureException { + String lc = endpoint.toLowerCase(); + if (!VALID_VERBS.contains(lc)) { + throw new LookupFailureException(String.format("%s is not a supported HTTP verb.", lc)); + } + } + protected String determineEndpoint(Map coordinates) { Map converted = coordinates.entrySet().stream() .collect(Collectors.toMap( @@ -399,15 +408,15 @@ private Request buildRequest(final String mimeType, final String method, final S } private void setAuthenticator(OkHttpClient.Builder okHttpClientBuilder, ConfigurationContext context) { - final String authUser = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_USERNAME).getValue()); + final String authUser = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_USERNAME).evaluateAttributeExpressions().getValue()); this.basicUser = authUser; isDigest = context.getProperty(PROP_DIGEST_AUTH).asBoolean(); + final String authPass = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_PASSWORD).evaluateAttributeExpressions().getValue()); + this.basicPass = authPass; // If the username/password properties are set then check if digest auth is being used if (!authUser.isEmpty() && isDigest) { - final String authPass = trimToEmpty(context.getProperty(PROP_BASIC_AUTH_PASSWORD).getValue()); - this.basicPass = authPass; /* * OkHttp doesn't have built-in Digest Auth Support. A ticket for adding it is here[1] but they authors decided instead to rely on a 3rd party lib. diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html index 62ab95a250f0..799d6a92ca5e 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html @@ -42,7 +42,7 @@

          Headers

          Headers are supported using dynamic properties. Just add a dynamic property and the name will be the header name and the value will be the value for the header. Expression language powered by input from the variable registry is supported.

          Dynamic URLs

          -

          The URL property supports expression language in a non-standard way: through the lookup key/value pairs configured on the processor. The configuration specified by the user will be passed +

          The URL property supports expression language through the lookup key/value pairs configured on the processor. The configuration specified by the user will be passed through to the expression language engine for evaluation. Note: flowfile attributes and variable registry settings will be disregarded here for this property.

          Ex. URL: http://example.com/service/${user.name}/friend/${friend.id}, combined with example record paths:

            From 5384129ba6ee91c14fb28773464c114d787c8665 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Fri, 8 Jun 2018 08:35:55 -0400 Subject: [PATCH 17/22] NIFI-5214 Renamed nifi-standard-web-utils to reflect that it is for tests. --- .../nifi-slack-bundle/nifi-slack-processors/pom.xml | 2 +- nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/pom.xml | 2 +- .../nifi-standard-bundle/nifi-standard-processors/pom.xml | 2 +- .../pom.xml | 2 +- .../src/main/java/org/apache/nifi/web/util/TestServer.java | 0 nifi-nar-bundles/nifi-standard-bundle/pom.xml | 2 +- .../nifi-lookup-services-bundle/nifi-lookup-services/pom.xml | 2 +- 7 files changed, 6 insertions(+), 6 deletions(-) rename nifi-nar-bundles/nifi-standard-bundle/{nifi-standard-web-utils => nifi-standard-web-test-utils}/pom.xml (96%) rename nifi-nar-bundles/nifi-standard-bundle/{nifi-standard-web-utils => nifi-standard-web-test-utils}/src/main/java/org/apache/nifi/web/util/TestServer.java (100%) diff --git a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/pom.xml b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/pom.xml index a82f425e5b53..1d8c6799f4f3 100644 --- a/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/pom.xml +++ b/nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/pom.xml @@ -89,7 +89,7 @@ org.apache.nifi - nifi-standard-web-utils + nifi-standard-web-test-utils 1.7.0-SNAPSHOT test diff --git a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/pom.xml b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/pom.xml index 354f7322f77e..70f47f2e01dd 100644 --- a/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/pom.xml +++ b/nifi-nar-bundles/nifi-spark-bundle/nifi-livy-processors/pom.xml @@ -94,7 +94,7 @@ org.apache.nifi - nifi-standard-web-utils + nifi-standard-web-test-utils 1.7.0-SNAPSHOT test diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml index 6d084d7c161e..67c511246f75 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml @@ -323,7 +323,7 @@ org.apache.nifi - nifi-standard-web-utils + nifi-standard-web-test-utils 1.7.0-SNAPSHOT test diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/pom.xml similarity index 96% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/pom.xml rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/pom.xml index 47774e87cf72..95703e031543 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/pom.xml @@ -16,7 +16,7 @@ language governing permissions and limitations under the License. --> 1.7.0-SNAPSHOT 4.0.0 - nifi-standard-web-utils + nifi-standard-web-test-utils org.apache.nifi diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/src/main/java/org/apache/nifi/web/util/TestServer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/src/main/java/org/apache/nifi/web/util/TestServer.java similarity index 100% rename from nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-utils/src/main/java/org/apache/nifi/web/util/TestServer.java rename to nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/src/main/java/org/apache/nifi/web/util/TestServer.java diff --git a/nifi-nar-bundles/nifi-standard-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/pom.xml index acfb6cdbdce0..b80e4c914b78 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/pom.xml @@ -30,7 +30,7 @@ nifi-standard-nar nifi-jolt-transform-json-ui nifi-standard-utils - nifi-standard-web-utils + nifi-standard-web-test-utils 2.9.5 diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml index 635bc3b0821b..28ce3df57703 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml @@ -137,7 +137,7 @@ org.apache.nifi - nifi-standard-web-utils + nifi-standard-web-test-utils 1.7.0-SNAPSHOT test From d4706efa1d9cf5f1202db6e2bba9daa5d4f69212 Mon Sep 17 00:00:00 2001 From: Koji Kawamura Date: Tue, 12 Jun 2018 15:57:43 +0900 Subject: [PATCH 18/22] NIFI-5214: Refactor RestLookupService. 1. Added 'Base URL' property to address environment specific part of URL. 2. Removed 'Record Path Property Name' property, because the name of a resulted record field of a record path can be obtained by field name. 3. Lower cased HTTP method name should be used throughout. 4. Added mimeType require check when body is specified. 5. Added debug log to print HTTP response code. 6. Prepare for NIFI-5287. 7. Fixed that mime.type being used regardless of whether body is specified or not, caused NullPointerException when 'mime.type' is not specified when it is not required. 8. Updated documentation. --- .../org/apache/nifi/web/util/TestServer.java | 4 +- .../apache/nifi/lookup/RestLookupService.java | 101 +++++++++++------- .../additionalDetails.html | 24 ++++- .../nifi/lookup/RestLookupServiceIT.groovy | 79 +++++++------- .../nifi/lookup/TestRestLookupService.groovy | 1 - .../lookup/rest/handlers/BasicAuth.groovy | 2 +- 6 files changed, 126 insertions(+), 85 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/src/main/java/org/apache/nifi/web/util/TestServer.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/src/main/java/org/apache/nifi/web/util/TestServer.java index 47bbdeacf8da..5fc74a5cfe29 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/src/main/java/org/apache/nifi/web/util/TestServer.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-web-test-utils/src/main/java/org/apache/nifi/web/util/TestServer.java @@ -135,14 +135,14 @@ public void shutdownServer() throws Exception { jetty.destroy(); } - private int getPort() { + public int getPort() { if (!jetty.isStarted()) { throw new IllegalStateException("Jetty server not started"); } return ((ServerConnector) jetty.getConnectors()[0]).getLocalPort(); } - private int getSecurePort() { + public int getSecurePort() { if (!jetty.isStarted()) { throw new IllegalStateException("Jetty server not started"); } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index 08172cdb833a..1488bcc6db56 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -27,6 +27,7 @@ import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; +import okhttp3.ResponseBody; import org.apache.nifi.annotation.behavior.DynamicProperties; import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.documentation.CapabilityDescription; @@ -82,6 +83,18 @@ "as the header name and the value as the header value.") }) public class RestLookupService extends AbstractControllerService implements RecordLookupService { + static final PropertyDescriptor BASE_URL = new PropertyDescriptor.Builder() + .name("rest-lookup-base-url") + .displayName("Base URL") + .description("The base URL for the REST endpoint. Expression language is evaluated against variable registry." + + " This property can be used to resolve environment specific part of the URL." + + " The result string is prepended to the 'URL'." + + " See 'Additional Details' to see an example.") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) + .required(false) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .build(); + static final PropertyDescriptor URL = new PropertyDescriptor.Builder() .name("rest-lookup-url") .displayName("URL") @@ -111,16 +124,6 @@ public class RestLookupService extends AbstractControllerService implements Reco .required(false) .build(); - static final PropertyDescriptor RECORD_PATH_PROPERTY_NAME = new PropertyDescriptor.Builder() - .name("rest-lookup-record-path-name") - .displayName("Record Path Property Name") - .description("An optional name for the property loaded by the record path. This will be the key used for the value " + - "when the loaded record is merged into record to be enriched. See docs for more details.") - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .addValidator(Validator.VALID) - .required(false) - .build(); - static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() .name("rest-lookup-ssl-context-service") .displayName("SSL Context Service") @@ -172,10 +175,10 @@ public class RestLookupService extends AbstractControllerService implements Reco static { DESCRIPTORS = Collections.unmodifiableList(Arrays.asList( + BASE_URL, URL, RECORD_READER, RECORD_PATH, - RECORD_PATH_PROPERTY_NAME, SSL_CONTEXT_SERVICE, PROXY_CONFIGURATION_SERVICE, PROP_BASIC_AUTH_USERNAME, @@ -195,7 +198,6 @@ protected List getSupportedPropertyDescriptors() { private OkHttpClient client; private Map headers; private volatile PreparedQuery compiledQuery; - private volatile String recordPathName; private volatile String basicUser; private volatile String basicPass; private volatile boolean isDigest; @@ -227,13 +229,12 @@ public void onEnabled(final ConfigurationContext context) { recordPath = RecordPath.compile(path); } - recordPathName = context.getProperty(RECORD_PATH_PROPERTY_NAME).isSet() - ? context.getProperty(RECORD_PATH_PROPERTY_NAME).evaluateAttributeExpressions().getValue() - : null; - getHeaders(context); - compiledQuery = Query.prepare(context.getProperty(URL).getValue()); + final String url = context.getProperty(URL).getValue(); + compiledQuery = context.getProperty(BASE_URL).isSet() + ? Query.prepare(context.getProperty(BASE_URL).evaluateAttributeExpressions().getValue() + url) + : Query.prepare(url); } @OnDisabled @@ -274,23 +275,49 @@ private void setProxy(OkHttpClient.Builder builder) { public Optional lookup(Map coordinates) throws LookupFailureException { final String endpoint = determineEndpoint(coordinates); final String mimeType = (String)coordinates.get(MIME_TYPE_KEY); - final String method = ((String)coordinates.getOrDefault(METHOD_KEY, "get")).trim(); + final String method = ((String)coordinates.getOrDefault(METHOD_KEY, "get")).trim().toLowerCase(); final String body = (String)coordinates.get(BODY_KEY); validateVerb(method); - if (StringUtils.isBlank(body) && (method.equals("post") || method.equals("put"))) { - throw new LookupFailureException( - String.format("Used HTTP verb %s without specifying the %s key to provide a payload.", method, BODY_KEY) - ); + if (StringUtils.isBlank(body)) { + if (method.equals("post") || method.equals("put")) { + throw new LookupFailureException( + String.format("Used HTTP verb %s without specifying the %s key to provide a payload.", method, BODY_KEY) + ); + } + } else { + if (StringUtils.isBlank(mimeType)) { + throw new LookupFailureException( + String.format("Request body is specified without its %s.", MIME_TYPE_KEY) + ); + } } Request request = buildRequest(mimeType, method, body, endpoint); try { Response response = executeRequest(request); - InputStream is = response.body().byteStream(); - Record record = handleResponse(is, coordinates); + if (getLogger().isDebugEnabled()) { + getLogger().debug("Response code {} was returned for coordinate {}", + new Object[]{response.code(), coordinates}); + } + + final ResponseBody responseBody = response.body(); + if (responseBody == null) { + return Optional.empty(); + } + + InputStream is = responseBody.byteStream(); + + // TODO: Need to use 'context' instead of 'coordinates', see NIFI-5287, this map conversion will not be needed. + final Map context = coordinates.entrySet().stream() + .filter(e -> e.getValue() != null) + .collect(Collectors.toMap( + e -> e.getKey(), + e -> e.getValue().toString() + )); + Record record = handleResponse(is, context); return Optional.ofNullable(record); } catch (Exception e) { @@ -299,15 +326,15 @@ public Optional lookup(Map coordinates) throws LookupFai } } - protected void validateVerb(String endpoint) throws LookupFailureException { - String lc = endpoint.toLowerCase(); - if (!VALID_VERBS.contains(lc)) { - throw new LookupFailureException(String.format("%s is not a supported HTTP verb.", lc)); + protected void validateVerb(String method) throws LookupFailureException { + if (!VALID_VERBS.contains(method)) { + throw new LookupFailureException(String.format("%s is not a supported HTTP verb.", method)); } } protected String determineEndpoint(Map coordinates) { Map converted = coordinates.entrySet().stream() + .filter(e -> e.getValue() != null) .collect(Collectors.toMap( e -> e.getKey(), e -> e.getValue().toString() @@ -329,13 +356,9 @@ protected Response executeRequest(Request request) throws IOException { return client.newCall(request).execute(); } - private Record handleResponse(InputStream is, Map coordinates) throws SchemaNotFoundException, MalformedRecordException, IOException { - Map variables = coordinates.entrySet().stream() - .collect(Collectors.toMap( - e -> e.getKey(), - e -> e.getValue().toString() - )); - try (RecordReader reader = readerFactory.createRecordReader(variables, is, getLogger())) { + private Record handleResponse(InputStream is, Map context) throws SchemaNotFoundException, MalformedRecordException, IOException { + + try (RecordReader reader = readerFactory.createRecordReader(context, is, getLogger())) { Record record = reader.nextRecord(); @@ -343,7 +366,7 @@ private Record handleResponse(InputStream is, Map coordinates) t Optional fv = recordPath.evaluate(record).getSelectedFields().findFirst(); if (fv.isPresent()) { FieldValue fieldValue = fv.get(); - RecordSchema schema = new SimpleRecordSchema(Arrays.asList(fieldValue.getField())); + RecordSchema schema = new SimpleRecordSchema(Collections.singletonList(fieldValue.getField())); Record temp; Object value = fieldValue.getValue(); @@ -353,7 +376,7 @@ private Record handleResponse(InputStream is, Map coordinates) t temp = new MapRecord(schema, (Map) value); } else { Map val = new HashMap<>(); - val.put(recordPathName, value); + val.put(fieldValue.getField().getFieldName(), value); temp = new MapRecord(schema, val); } @@ -371,14 +394,14 @@ record = null; } private Request buildRequest(final String mimeType, final String method, final String body, final String endpoint) { - final MediaType mt = MediaType.parse(mimeType); RequestBody requestBody = null; if (body != null) { + final MediaType mt = MediaType.parse(mimeType); requestBody = RequestBody.create(mt, body); } Request.Builder request = new Request.Builder() .url(endpoint); - switch(method.toLowerCase()) { + switch(method) { case "delete": request = body != null ? request.delete(requestBody) : request.delete(); break; diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html index 799d6a92ca5e..bf3475194597 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html @@ -21,10 +21,9 @@

            General

            -

            This lookup service has the following required keys:

            +

            This lookup service has the following optional lookup coordinate keys:

              -
            • mime.type
              • -
            • request.method; valid values: +
            • request.method; defaults to 'get', valid values:
              • delete
              • get
                • @@ -32,9 +31,11 @@

                General

              • put
              • +
            • body; contains a string representing JSON, XML, etc. to be sent with any + of those methods except for "get".
              • +
            • mime.type; specifies media type of the request body, required when 'body' is passed.
              • +
            • *; any other keys can be configured to pass variables to resolve target URLs. See 'Dynamic URLs' section below.
            -

            In addition to the required keys, a key "body" can be added which contains a string representing JSON, XML, etc. to be sent with any - of those methods except for "get."

            The record reader is used to consume the response of the REST service call and turn it into one or more records. The record path property is provided to allow for a lookup path to either a nested record or a single point deep in the REST response. Note: a valid schema must be built that encapsulates the REST response accurately in order for this service to work.

            @@ -50,5 +51,18 @@

            Dynamic URLs

          • friend.id => "/example/first_friend"

          Would dynamically produce an endpoint of http://example.com/service/john.smith/friend/12345

          + +

          Environment specific URLs

          + +

          If target URL has environment specific part, such as server name and port, + then 'Base URL' property can be used to resolve it by an expression language with variable registry. + This way, the consistent lookup coordinate keys can be used in different environments.

          +

          Ex. Base URL: http://${apiServerHostname}:${apiServerPort}, combined with example variable registry:

          +
            +
          • apiServerHostname => "test.example.com"
            • +
          • apiServerPort => "8080"
            • +
          +

          Ex. URL: /service/${user.name}/friend/${friend.id}, combined with the previous example record paths

          +

          Would dynamically produce an endpoint of http://test.example.com:8080/service/john.smith/friend/12345

          \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index 802097e16f00..455be6954a8a 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -48,6 +48,7 @@ class RestLookupServiceIT { static final RecordSchema simpleSchema static final RecordSchema nestedSchema + TestServer server TestRunner runner RestLookupService lookupService @@ -71,27 +72,23 @@ class RestLookupServiceIT { runner.setProperty(reader, SchemaAccessUtils.SCHEMA_REGISTRY, "registry") runner.setProperty(lookupService, SchemaAccessUtils.SCHEMA_REGISTRY, "registry") runner.setProperty(lookupService, RestLookupService.RECORD_READER, "jsonReader") + runner.setProperty(lookupService, RestLookupService.BASE_URL, 'http://localhost:${serverPort}') runner.setProperty(TestRestLookupServiceProcessor.CLIENT_SERVICE, "lookupService") runner.enableControllerService(registry) runner.enableControllerService(reader) - runner.enableControllerService(lookupService) - - runner.assertValid() } @Test void basicAuth() { - runner.disableControllerService(lookupService) runner.setProperty(lookupService, RestLookupService.PROP_BASIC_AUTH_USERNAME, "john.smith") runner.setProperty(lookupService, RestLookupService.PROP_BASIC_AUTH_PASSWORD, "testing1234") - runner.enableControllerService(lookupService) TestServer server = new TestServer() server.addHandler(new BasicAuth()) try { server.startServer() - setEndpoint(server.url, "/simple") + setEndpoint(server.port, "/simple") def coordinates = [ "schema.name": "simple", @@ -133,7 +130,7 @@ class RestLookupServiceIT { try { server.startServer() - setEndpoint(server.url, "/simple") + setEndpoint(server.port, "/simple") def coordinates = [ "schema.name": "simple", @@ -160,7 +157,7 @@ class RestLookupServiceIT { try { server.startServer() - setEndpoint(server.url, "/simple") + setEndpoint(server.port, "/simple") def coordinates = [ "schema.name": "simple", @@ -187,7 +184,7 @@ class RestLookupServiceIT { try { server.startServer() - setEndpoint(server.url, "/simple_array") + setEndpoint(server.port, "/simple_array") def coordinates = [ "schema.name": "simple", @@ -207,10 +204,8 @@ class RestLookupServiceIT { @Test void testHeaders() { - runner.disableControllerService(lookupService) runner.setProperty(lookupService, "X-USER", "jane.doe") runner.setProperty(lookupService, "X-PASS", "testing7890") - runner.enableControllerService(lookupService) TestServer server = new TestServer() ServletHandler handler = new ServletHandler() @@ -219,7 +214,7 @@ class RestLookupServiceIT { try { server.startServer() - setEndpoint(server.url, "/simple") + setEndpoint(server.port, "/simple") def coordinates = [ "schema.name": "simple", @@ -239,9 +234,7 @@ class RestLookupServiceIT { @Test void complexJson() { - runner.disableControllerService(lookupService) runner.setProperty(lookupService, RestLookupService.RECORD_PATH, "/top/middle/inner") - runner.enableControllerService(lookupService) TestServer server = new TestServer() ServletHandler handler = new ServletHandler() @@ -250,7 +243,7 @@ class RestLookupServiceIT { try { server.startServer() - setEndpoint(server.url, "/complex") + setEndpoint(server.port, "/complex") def coordinates = [ "schema.name": "complex", @@ -276,17 +269,14 @@ class RestLookupServiceIT { handler.addServletWithMapping(VerbTest.class, "/simple") server.addHandler(handler) try { - runner.disableControllerService(lookupService) - runner.setProperty(lookupService, "needs-body", "true") - runner.enableControllerService(lookupService) server.startServer() - setEndpoint(server.url, "/simple") + setEndpoint(server.port, "/simple") - def validation = { String verb, boolean addBody -> + def validation = { String verb, boolean addBody, boolean addMimeType, boolean valid -> def coordinates = [ "schema.name" : "simple", - "mime.type" : "application/json", + "mime.type" : addMimeType ? "application/json" : null, "request.method": verb ] @@ -294,22 +284,34 @@ class RestLookupServiceIT { coordinates["request.body"] = prettyPrint(toJson([ msg: "Hello, world" ])) } - Optional response = lookupService.lookup(coordinates) - Assert.assertTrue(response.isPresent()) - def record = response.get() - Assert.assertEquals("john.smith", record.getAsString("username")) - Assert.assertEquals("testing1234", record.getAsString("password")) + try { + Optional response = lookupService.lookup(coordinates) + if (!valid) { + Assert.fail("Validation should fail.") + } + + Assert.assertTrue(response.isPresent()) + def record = response.get() + Assert.assertEquals("john.smith", record.getAsString("username")) + Assert.assertEquals("testing1234", record.getAsString("password")) + + } catch (LookupFailureException e) { + if (valid) { + Assert.fail("Validation should be successful.") + } + } } - ["delete", "post", "put"].each { verb -> - validation(verb, true) - } + // Delete does not require body nor mimeType. + validation("delete", false, false, true) - runner.disableControllerService(lookupService) - runner.setProperty(lookupService, "needs-body", "") - runner.enableControllerService(lookupService) + // Post and Put require body and mimeType. + ["post", "put"].each { verb -> + validation(verb, false, false, false) + validation(verb, true, false, false) + validation(verb, true, true, true) + } - validation("delete", false) } finally { server.shutdownServer() } @@ -324,7 +326,7 @@ class RestLookupServiceIT { try { server.startServer() - setEndpoint(server.url, '/simple/${user.name}/friends/${friend.id}') + setEndpoint(server.port, '/simple/${user.name}/friends/${friend.id}') def coordinates = [ "schema.name": "simple", @@ -345,9 +347,12 @@ class RestLookupServiceIT { } } - void setEndpoint(String serverUrl, String endpoint) { - runner.disableControllerService(lookupService) - runner.setProperty(lookupService, RestLookupService.URL, serverUrl + endpoint) + void setEndpoint(Integer serverPort, String endpoint) { + // Resolve environmental part of the URL via variable registry. + runner.setVariable("serverPort", String.valueOf(serverPort)) + runner.setProperty(lookupService, RestLookupService.URL, endpoint) runner.enableControllerService(lookupService) + + runner.assertValid() } } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy index 5112ac46b8c5..0de379b093f3 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/TestRestLookupService.groovy @@ -112,7 +112,6 @@ class TestRestLookupService { runner.disableControllerService(lookupService) runner.setProperty(lookupService, RestLookupService.RECORD_PATH, "/person/sport") - runner.setProperty(lookupService, RestLookupService.RECORD_PATH_PROPERTY_NAME, "sport") runner.enableControllerService(lookupService) runner.assertValid() diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy index c0c833e67263..eb0c6f0510fc 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/handlers/BasicAuth.groovy @@ -36,7 +36,7 @@ class BasicAuth extends AbstractHandler { def headers = [] request.headerNames.each { headers << it } - if (!authString || authString != "Basic am9obi5zbWl0aDpudWxs") { + if (!authString || authString != "Basic am9obi5zbWl0aDp0ZXN0aW5nMTIzNA==") { response.status = 401 response.setHeader("WWW-Authenticate", "Basic realm=\"Jetty\"") response.setHeader("response.phrase", "Unauthorized") From 0be0ccaf4c0e15823c1cdc8c1b4d03cd96eb7f8a Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Tue, 12 Jun 2018 08:09:46 -0400 Subject: [PATCH 19/22] NIFI-5214 Fixed AWS processors that broke w/ change to TestServer location. --- nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/pom.xml | 3 +-- .../nifi/processors/aws/wag/TestInvokeAWSGatewayApiCommon.java | 2 +- .../TestInvokeInvokeAmazonGatewayApiWithControllerService.java | 2 +- .../aws/wag/TestInvokeInvokeAmazonGatewayApiWithCredFile.java | 2 +- .../wag/TestInvokeInvokeAmazonGatewayApiWithStaticAuth.java | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/pom.xml b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/pom.xml index db2b98939510..a1ba7757c488 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/pom.xml +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/pom.xml @@ -52,9 +52,8 @@ org.apache.nifi - nifi-standard-processors + nifi-standard-web-test-utils 1.7.0-SNAPSHOT - test-jar test diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeAWSGatewayApiCommon.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeAWSGatewayApiCommon.java index a071b0e04f1e..f230cf1ca078 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeAWSGatewayApiCommon.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeAWSGatewayApiCommon.java @@ -34,12 +34,12 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processors.aws.AbstractAWSProcessor; import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderControllerService; -import org.apache.nifi.processors.standard.TestServer; import org.apache.nifi.provenance.ProvenanceEventRecord; import org.apache.nifi.provenance.ProvenanceEventType; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; +import org.apache.nifi.web.util.TestServer; import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithControllerService.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithControllerService.java index 9dd2f8eccaa1..403396ba6b3a 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithControllerService.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithControllerService.java @@ -17,8 +17,8 @@ package org.apache.nifi.processors.aws.wag; import java.io.IOException; -import org.apache.nifi.processors.standard.TestServer; import org.apache.nifi.util.TestRunners; +import org.apache.nifi.web.util.TestServer; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithCredFile.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithCredFile.java index e33c6b34c29c..822ea76be771 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithCredFile.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithCredFile.java @@ -17,8 +17,8 @@ package org.apache.nifi.processors.aws.wag; import java.io.IOException; -import org.apache.nifi.processors.standard.TestServer; import org.apache.nifi.util.TestRunners; +import org.apache.nifi.web.util.TestServer; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithStaticAuth.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithStaticAuth.java index 6bb5232a8f5a..43dd7924ae02 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithStaticAuth.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/wag/TestInvokeInvokeAmazonGatewayApiWithStaticAuth.java @@ -17,8 +17,8 @@ package org.apache.nifi.processors.aws.wag; import java.io.IOException; -import org.apache.nifi.processors.standard.TestServer; import org.apache.nifi.util.TestRunners; +import org.apache.nifi.web.util.TestServer; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; From 1687b5eccbfcba006d7528706a51b35e16070f81 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Wed, 13 Jun 2018 20:36:57 -0400 Subject: [PATCH 20/22] NIFI-5214 Added changes requested in a code review. --- .../nifi-lookup-services/pom.xml | 2 + .../apache/nifi/lookup/RestLookupService.java | 17 +++--- .../apache/nifi/lookup/rest/SchemaUtil.groovy | 54 +------------------ .../src/test/resources/complex.avsc | 36 +++++++++++++ .../src/test/resources/simple.avsc | 14 +++++ 5 files changed, 63 insertions(+), 60 deletions(-) create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/complex.avsc create mode 100644 nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/simple.avsc diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml index 28ce3df57703..29b6435d8ea7 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/pom.xml @@ -160,6 +160,8 @@ apache-rat-plugin + src/test/resources/complex.avsc + src/test/resources/simple.avsc src/test/resources/test.csv src/test/resources/test_Windows-31J.csv src/test/resources/test.properties diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index 1488bcc6db56..0465a9ede244 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -77,7 +77,7 @@ import static org.apache.commons.lang3.StringUtils.trimToEmpty; @Tags({ "rest", "lookup", "json", "xml", "http" }) -@CapabilityDescription("Use a REST service to enrich records.") +@CapabilityDescription("Use a REST service to look up values.") @DynamicProperties({ @DynamicProperty(name = "*", value = "*", description = "All dynamic properties are added as HTTP headers with the name " + "as the header name and the value as the header value.") @@ -192,11 +192,11 @@ protected List getSupportedPropertyDescriptors() { return DESCRIPTORS; } - private ProxyConfigurationService proxyConfigurationService; - private RecordReaderFactory readerFactory; - private RecordPath recordPath; - private OkHttpClient client; - private Map headers; + private volatile ProxyConfigurationService proxyConfigurationService; + private volatile RecordReaderFactory readerFactory; + private volatile RecordPath recordPath; + private volatile OkHttpClient client; + private volatile Map headers; private volatile PreparedQuery compiledQuery; private volatile String basicUser; private volatile String basicPass; @@ -229,7 +229,7 @@ public void onEnabled(final ConfigurationContext context) { recordPath = RecordPath.compile(path); } - getHeaders(context); + buildHeaders(context); final String url = context.getProperty(URL).getValue(); compiledQuery = context.getProperty(BASE_URL).isSet() @@ -239,10 +239,11 @@ public void onEnabled(final ConfigurationContext context) { @OnDisabled public void onDisabled() { + this.recordPath = null; this.compiledQuery = null; } - private void getHeaders(ConfigurationContext context) { + private void buildHeaders(ConfigurationContext context) { headers = new HashMap<>(); for (PropertyDescriptor descriptor : context.getProperties().keySet()) { if (descriptor.isDynamic()) { diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy index a152f90d48a7..8ac4e4b45943 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/rest/SchemaUtil.groovy @@ -18,57 +18,7 @@ package org.apache.nifi.lookup.rest class SchemaUtil { - static final String SIMPLE = """ - { - "type": "record", - "name": "SimpleRecord", - "fields": [ - { - "name": "username", - "type": "string" - }, - { - "name": "password", - "type": "string" - } - ] - } - """ + static final String SIMPLE = SchemaUtil.class.getResourceAsStream("/simple.avsc").text - static final String COMPLEX = """{ - "type": "record", - "name": "ComplexRecord", - "fields": [ - { - "name": "top", - "type": { - "type": "record", - "name": "TopRecord", - "fields": [ - { - "name": "middle", - "type": { - "name": "MiddleRecord", - "type": "record", - "fields": [ - { - "name": "inner", - "type": { - "type": "record", - "name": "InnerRecord", - "fields": [ - { "name": "username", "type": "string" }, - { "name": "password", "type": "string" }, - { "name": "email", "type": "string" } - ] - } - } - ] - } - } - ] - } - } - ] - }""" + static final String COMPLEX = SchemaUtil.class.getResourceAsStream("/complex.avsc").text } diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/complex.avsc b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/complex.avsc new file mode 100644 index 000000000000..96baf3a8a83c --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/complex.avsc @@ -0,0 +1,36 @@ +{ + "type": "record", + "name": "ComplexRecord", + "fields": [ + { + "name": "top", + "type": { + "type": "record", + "name": "TopRecord", + "fields": [ + { + "name": "middle", + "type": { + "name": "MiddleRecord", + "type": "record", + "fields": [ + { + "name": "inner", + "type": { + "type": "record", + "name": "InnerRecord", + "fields": [ + { "name": "username", "type": "string" }, + { "name": "password", "type": "string" }, + { "name": "email", "type": "string" } + ] + } + } + ] + } + } + ] + } + } + ] +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/simple.avsc b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/simple.avsc new file mode 100644 index 000000000000..cb1bd14df9ec --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/resources/simple.avsc @@ -0,0 +1,14 @@ +{ + "type": "record", + "name": "SimpleRecord", + "fields": [ + { + "name": "username", + "type": "string" + }, + { + "name": "password", + "type": "string" + } + ] +} \ No newline at end of file From 0853d375ab320f8bc18519f12319b8c6a806966e Mon Sep 17 00:00:00 2001 From: Koji Kawamura Date: Wed, 13 Jun 2018 09:46:51 +0900 Subject: [PATCH 21/22] Refactored the way to evaluate EL for URL property - Use PropertyValue instead of PreparedQuery to utilize Variable Registry. - Removed BASE_URL because Variable Registry can be used at URL --- .../apache/nifi/lookup/RestLookupService.java | 31 +++++-------------- .../additionalDetails.html | 15 ++++----- .../nifi/lookup/RestLookupServiceIT.groovy | 3 +- 3 files changed, 14 insertions(+), 35 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index 0465a9ede244..41876a851e12 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -34,9 +34,8 @@ import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnDisabled; import org.apache.nifi.annotation.lifecycle.OnEnabled; -import org.apache.nifi.attribute.expression.language.PreparedQuery; -import org.apache.nifi.attribute.expression.language.Query; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.Validator; import org.apache.nifi.controller.AbstractControllerService; import org.apache.nifi.controller.ConfigurationContext; @@ -83,24 +82,12 @@ "as the header name and the value as the header value.") }) public class RestLookupService extends AbstractControllerService implements RecordLookupService { - static final PropertyDescriptor BASE_URL = new PropertyDescriptor.Builder() - .name("rest-lookup-base-url") - .displayName("Base URL") - .description("The base URL for the REST endpoint. Expression language is evaluated against variable registry." + - " This property can be used to resolve environment specific part of the URL." + - " The result string is prepended to the 'URL'." + - " See 'Additional Details' to see an example.") - .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) - .required(false) - .addValidator(StandardValidators.NON_BLANK_VALIDATOR) - .build(); - static final PropertyDescriptor URL = new PropertyDescriptor.Builder() .name("rest-lookup-url") .displayName("URL") .description("The URL for the REST endpoint. Expression language is evaluated against the lookup key/value pairs, " + - "not flowfile attributes or variable registry.") - .expressionLanguageSupported(ExpressionLanguageScope.NONE) + "not flowfile attributes.") + .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY) .required(true) .addValidator(StandardValidators.NON_BLANK_VALIDATOR) .build(); @@ -175,7 +162,6 @@ public class RestLookupService extends AbstractControllerService implements Reco static { DESCRIPTORS = Collections.unmodifiableList(Arrays.asList( - BASE_URL, URL, RECORD_READER, RECORD_PATH, @@ -197,7 +183,7 @@ protected List getSupportedPropertyDescriptors() { private volatile RecordPath recordPath; private volatile OkHttpClient client; private volatile Map headers; - private volatile PreparedQuery compiledQuery; + private volatile PropertyValue urlTemplate; private volatile String basicUser; private volatile String basicPass; private volatile boolean isDigest; @@ -231,16 +217,13 @@ public void onEnabled(final ConfigurationContext context) { buildHeaders(context); - final String url = context.getProperty(URL).getValue(); - compiledQuery = context.getProperty(BASE_URL).isSet() - ? Query.prepare(context.getProperty(BASE_URL).evaluateAttributeExpressions().getValue() + url) - : Query.prepare(url); + urlTemplate = context.getProperty(URL); } @OnDisabled public void onDisabled() { this.recordPath = null; - this.compiledQuery = null; + this.urlTemplate = null; } private void buildHeaders(ConfigurationContext context) { @@ -340,7 +323,7 @@ protected String determineEndpoint(Map coordinates) { e -> e.getKey(), e -> e.getValue().toString() )); - return compiledQuery.evaluateExpressions(converted, null); + return urlTemplate.evaluateAttributeExpressions(converted).getValue(); } protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html index bf3475194597..b38b919bd638 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/resources/docs/org.apache.nifi.lookup.RestLookupService/additionalDetails.html @@ -43,26 +43,23 @@

          Headers

          Headers are supported using dynamic properties. Just add a dynamic property and the name will be the header name and the value will be the value for the header. Expression language powered by input from the variable registry is supported.

          Dynamic URLs

          -

          The URL property supports expression language through the lookup key/value pairs configured on the processor. The configuration specified by the user will be passed - through to the expression language engine for evaluation. Note: flowfile attributes and variable registry settings will be disregarded here for this property.

          -

          Ex. URL: http://example.com/service/${user.name}/friend/${friend.id}, combined with example record paths:

          +

          The URL property supports expression language through the lookup key/value pairs configured on the component using this lookup service (e.g. LookupRecord processor). The configuration specified by the user will be passed + through to the expression language engine for evaluation. Note: flowfile attributes will be disregarded here for this property.

          +

          Ex. URL: http://example.com/service/${user.name}/friend/${friend.id}, combined with example record paths at LookupRecord processor:

          • user.name => "/example/username"
          • friend.id => "/example/first_friend"

          Would dynamically produce an endpoint of http://example.com/service/john.smith/friend/12345

          -

          Environment specific URLs

          +

          Using Variable Registry with URLs

          -

          If target URL has environment specific part, such as server name and port, - then 'Base URL' property can be used to resolve it by an expression language with variable registry. - This way, the consistent lookup coordinate keys can be used in different environments.

          -

          Ex. Base URL: http://${apiServerHostname}:${apiServerPort}, combined with example variable registry:

          +

          In addition to the lookup key/value pairs, Variable Registry can be referred from expression languages configured at the URL property.

          +

          Ex. URL: http://${apiServerHostname}:${apiServerPort}/service/${user.name}/friend/${friend.id}, combined with the previous example record paths, and variable registry:

          • apiServerHostname => "test.example.com"
          • apiServerPort => "8080"
          -

          Ex. URL: /service/${user.name}/friend/${friend.id}, combined with the previous example record paths

          Would dynamically produce an endpoint of http://test.example.com:8080/service/john.smith/friend/12345

          \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index 455be6954a8a..cb08ba788edd 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -72,7 +72,6 @@ class RestLookupServiceIT { runner.setProperty(reader, SchemaAccessUtils.SCHEMA_REGISTRY, "registry") runner.setProperty(lookupService, SchemaAccessUtils.SCHEMA_REGISTRY, "registry") runner.setProperty(lookupService, RestLookupService.RECORD_READER, "jsonReader") - runner.setProperty(lookupService, RestLookupService.BASE_URL, 'http://localhost:${serverPort}') runner.setProperty(TestRestLookupServiceProcessor.CLIENT_SERVICE, "lookupService") runner.enableControllerService(registry) runner.enableControllerService(reader) @@ -350,7 +349,7 @@ class RestLookupServiceIT { void setEndpoint(Integer serverPort, String endpoint) { // Resolve environmental part of the URL via variable registry. runner.setVariable("serverPort", String.valueOf(serverPort)) - runner.setProperty(lookupService, RestLookupService.URL, endpoint) + runner.setProperty(lookupService, RestLookupService.URL, "http://localhost:${serverPort}" + endpoint) runner.enableControllerService(lookupService) runner.assertValid() From 1919d2ee125fb344bf81b4c745e2aaff561624c1 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Thu, 14 Jun 2018 06:57:15 -0400 Subject: [PATCH 22/22] NIFI-5214 Rebased to use new LookupService method. --- .../apache/nifi/lookup/RestLookupService.java | 14 +++---- .../nifi/lookup/RestLookupServiceIT.groovy | 40 +++++++++++-------- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java index 41876a851e12..656ad5f8d7ab 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/main/java/org/apache/nifi/lookup/RestLookupService.java @@ -255,8 +255,14 @@ private void setProxy(OkHttpClient.Builder builder) { } } + @Override public Optional lookup(Map coordinates) throws LookupFailureException { + return lookup(coordinates, null); + } + + @Override + public Optional lookup(Map coordinates, Map context) throws LookupFailureException { final String endpoint = determineEndpoint(coordinates); final String mimeType = (String)coordinates.get(MIME_TYPE_KEY); final String method = ((String)coordinates.getOrDefault(METHOD_KEY, "get")).trim().toLowerCase(); @@ -293,14 +299,6 @@ public Optional lookup(Map coordinates) throws LookupFai } InputStream is = responseBody.byteStream(); - - // TODO: Need to use 'context' instead of 'coordinates', see NIFI-5287, this map conversion will not be needed. - final Map context = coordinates.entrySet().stream() - .filter(e -> e.getValue() != null) - .collect(Collectors.toMap( - e -> e.getKey(), - e -> e.getValue().toString() - )); Record record = handleResponse(is, context); return Optional.ofNullable(record); diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy index cb08ba788edd..e2f9a61fec06 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy +++ b/nifi-nar-bundles/nifi-standard-services/nifi-lookup-services-bundle/nifi-lookup-services/src/test/groovy/org/apache/nifi/lookup/RestLookupServiceIT.groovy @@ -90,12 +90,13 @@ class RestLookupServiceIT { setEndpoint(server.port, "/simple") def coordinates = [ - "schema.name": "simple", "mime.type": "application/json", "request.method": "get" ] - Optional response = lookupService.lookup(coordinates) + def context = [ "schema.name": "simple" ] + + Optional response = lookupService.lookup(coordinates, context) Assert.assertTrue(response.isPresent()) def record = response.get() Assert.assertEquals("john.smith", record.getAsString("username")) @@ -132,12 +133,13 @@ class RestLookupServiceIT { setEndpoint(server.port, "/simple") def coordinates = [ - "schema.name": "simple", "mime.type": "application/json", "request.method": "get" ] - Optional response = lookupService.lookup(coordinates) + def context = [ "schema.name": "simple" ] + + Optional response = lookupService.lookup(coordinates, context) Assert.assertTrue(response.isPresent()) def record = response.get() Assert.assertEquals("john.smith", record.getAsString("username")) @@ -159,12 +161,13 @@ class RestLookupServiceIT { setEndpoint(server.port, "/simple") def coordinates = [ - "schema.name": "simple", "mime.type": "application/json", "request.method": "get" ] - Optional response = lookupService.lookup(coordinates) + def context = [ "schema.name": "simple" ] + + Optional response = lookupService.lookup(coordinates, context) Assert.assertTrue(response.isPresent()) def record = response.get() Assert.assertNull(record.getAsString("username")) @@ -186,12 +189,13 @@ class RestLookupServiceIT { setEndpoint(server.port, "/simple_array") def coordinates = [ - "schema.name": "simple", "mime.type": "application/json", "request.method": "get" ] - Optional response = lookupService.lookup(coordinates) + def context = [ "schema.name": "simple" ] + + Optional response = lookupService.lookup(coordinates, context) Assert.assertTrue(response.isPresent()) def record = response.get() Assert.assertEquals("john.smith", record.getAsString("username")) @@ -216,12 +220,13 @@ class RestLookupServiceIT { setEndpoint(server.port, "/simple") def coordinates = [ - "schema.name": "simple", "mime.type": "application/json", "request.method": "get" ] - Optional response = lookupService.lookup(coordinates) + def context = [ "schema.name": "simple" ] + + Optional response = lookupService.lookup(coordinates, context) Assert.assertTrue(response.isPresent()) def record = response.get() Assert.assertEquals("jane.doe", record.getAsString("username")) @@ -245,12 +250,13 @@ class RestLookupServiceIT { setEndpoint(server.port, "/complex") def coordinates = [ - "schema.name": "complex", "mime.type": "application/json", "request.method": "get" ] - Optional response = lookupService.lookup(coordinates) + def context = [ "schema.name": "complex" ] + + Optional response = lookupService.lookup(coordinates, context) Assert.assertTrue(response.isPresent()) def record = response.get() Assert.assertEquals("jane.doe", record.getAsString("username")) @@ -274,17 +280,18 @@ class RestLookupServiceIT { def validation = { String verb, boolean addBody, boolean addMimeType, boolean valid -> def coordinates = [ - "schema.name" : "simple", "mime.type" : addMimeType ? "application/json" : null, "request.method": verb ] + def context = [ "schema.name": "simple" ] + if (addBody) { coordinates["request.body"] = prettyPrint(toJson([ msg: "Hello, world" ])) } try { - Optional response = lookupService.lookup(coordinates) + Optional response = lookupService.lookup(coordinates, context) if (!valid) { Assert.fail("Validation should fail.") } @@ -328,7 +335,6 @@ class RestLookupServiceIT { setEndpoint(server.port, '/simple/${user.name}/friends/${friend.id}') def coordinates = [ - "schema.name": "simple", "mime.type": "application/json", "request.method": "get", "user.name": "john.smith", @@ -336,7 +342,9 @@ class RestLookupServiceIT { "endpoint.template": true ] - Optional response = lookupService.lookup(coordinates) + def context = [ "schema.name": "simple" ] + + Optional response = lookupService.lookup(coordinates, context) Assert.assertTrue(response.isPresent()) def record = response.get() Assert.assertEquals("john.smith", record.getAsString("username"))