adapter: DataImports API and refactoring

adapter/README.md

@@ -4,8 +4,9 @@ The data coming from the external sources can be fetched over various protocols
 
 ## Concepts
 - **Datasource**: Description of a datasource. This description can be transformed into an *adapter* to import data from a data source and forward it to downstream services.
-- **Adapter**: Configuration to import data from a datasource; can be derived from a *datasource* config, or is provided by a user to generate a *preview*
-- **Preview**: Stateless preview that allows executing a datasource config once and synchronously returning the result of the import and interpretation; does not send the result to downstream services (difference tocreating and triggering a datasource).
+- **Adapter**: Configuration to import data from a datasource; can be derived from a *datasource* config, or is provided by a user to generate a *preview*.
+- **Preview**: Stateless preview that allows executing a datasource config once and synchronously returning the result of the import and interpretation; does not send the result to downstream services (difference to creating and triggering a datasource).
+- **Data import**: One execution of the import of a _datasource_. The result and metadata get stored in the database and can be accessed for each _datasource_.
 
 ## Current Features
 * Currently the adapter service is only a prototype and can handle JSON, XML and CSV files that can be fetched over HTTP.
@@ -47,23 +48,22 @@ Support for new protocols or data formats can easily be achieved by adding class
 | *base_url*/version  | GET  | -  | String containing the application version  |
 | *base_url*/formats  | GET  | -  | JsonArray of data formats available for parsing and possible parameters |
 | *base_url*/protocols  | GET  | -  | JsonArray of protocols available for importing and possible parameters  |
-| *base_url*/preview  | POST  | AdapterConfig | ImportResponse | 
-| *base_url*/preview/raw  | POST  | ProtocolConfig | ImportResponse | 
-| *base_url*/data/{id}  | GET  | -  | JSON representation of imported data with {id} |
+| *base_url*/preview  | POST  | AdapterConfig | PreviewResponse | 
+| *base_url*/preview/raw  | POST  | ProtocolConfig | PreviewResponse | 
 
 
 When started via docker-compose *base_url* is `http://localhost:9000/api/adapter`
 
 ### Adapter Config
 ```
 {
-    "protocol": ProtocolConfig, 
-    "format": {
-      "type": "JSON" | "XML" | "CSV",
-      "parameters": { } | CSVParameters
-    }
+  "protocol": ProtocolConfig, 
+  "format": {
+    "type": "JSON" | "XML" | "CSV",
+    "parameters": { } | CSVParameters
   }
-  ```
+}
+```
 
 ### Protocol Config
 ```
@@ -86,9 +86,11 @@ When started via docker-compose *base_url* is `http://localhost:9000/api/adapter
 }
 ```
 
-### ImportResponse
+### PreviewResponse
 ```
+{
     "data": <<Stringified JSON or RAW representation of payload>>
+}
 ```
 
 
@@ -101,7 +103,13 @@ When started via docker-compose *base_url* is `http://localhost:9000/api/adapter
 | *base_url*/datasources/{id}  | PUT  | Datasource Config  | Updated datasource with {id}  |
 | *base_url*/datasources  | DELETE  | -  | Delete all datasources  |
 | *base_url*/datasources/{id}  | DELETE  | -  | Delete datasource with {id} |
-| *base_url*/datasources/${id}/trigger  | POST  | Parameters  | Adapter ImportResponse |
+| *base_url*/datasources/{id}/trigger  | POST  | Parameters  | DataImport |
+| *base_url*/datasources/{id}/imports  | GET  | -  | All DataImports for datasource with {id} |
+| *base_url*/datasources/{id}/imports/{importId}  | GET  | -  | DataImports with {importId} for datasource with {id} |
+| *base_url*/datasources/{id}/imports/latest  | GET  | -  | Latest DataImport for datasource with {id} |
+| *base_url*/datasources/{id}/imports/{importId}/data  | GET  | -  | Actual data of DataImport with {importId} for datasource with {id} |
+| *base_url*/datasources/{id}/imports/latest/data  | GET  | -  | Actual data for latest DataImport for datasource with {id} |
+
 
 When started via docker-compose *base_url* is `http://localhost:9000/api/adapter`
 
@@ -159,10 +167,11 @@ When started via docker-compose *base_url* is `http://localhost:9000/api/adapter
 ### Metadata 
 ```
 {
-  "author":String,
-  "displayName":String,
-  "license":String,
-  "description":String
+  "author": String,
+  "displayName": String,
+  "license": String,
+  "description": String,
+  "creationTimestamp: Date (format: yyyy-MM-dd'T'HH:mm:ss.SSSXXX),
 }
 ```
 
@@ -179,3 +188,12 @@ When started via docker-compose *base_url* is `http://localhost:9000/api/adapter
   "parameters": <<Map of type <String, String> for open parameter to replace with the value>>
 }
 ```
+
+### DataImport
+```
+{
+  "id": Number,
+  "timestamp": Date (format: yyyy-MM-dd'T'HH:mm:ss.SSSXXX)
+  "location": String (relative URI)
+}
+```

adapter/integration-test/src/adapter-datasource-trigger.test.js

@@ -1,5 +1,3 @@
-/* eslint-env jest */
-// @ts-check
 const request = require('supertest')
 
 const {
@@ -52,16 +50,17 @@ describe('Datasource triggering', () => {
     expect(datasourceResponse.status).toEqual(201)
     const datasourceId = datasourceResponse.body.id
 
-    const dataMetaData = await request(ADAPTER_URL)
+    const dataImportMetaData = await request(ADAPTER_URL)
       .post(`/datasources/${datasourceId}/trigger`)
       .send(runtimeParameters)
-    expect(dataMetaData.status).toEqual(200)
-    expect(dataMetaData.body.id).toBeGreaterThanOrEqual(0)
-    const id = dataMetaData.body.id
-    expect(dataMetaData.body.location).toEqual(`/data/${id}`)
+    expect(dataImportMetaData.status).toEqual(200)
+    expect(dataImportMetaData.body.id).toBeGreaterThanOrEqual(0)
+    expect(dataImportMetaData.body.timestamp).toBeTruthy()
+    const dataImportId = dataImportMetaData.body.id
+    expect(dataImportMetaData.body.location).toEqual(`/datasources/${datasourceId}/imports/${dataImportId}/data`)
 
     const data = await request(ADAPTER_URL)
-      .get(`/data/${id}`)
+      .get(dataImportMetaData.body.location)
       .send()
     expect(data.status).toEqual(200)
     expect(data.body.id).toBe(runtimeParameters.parameters.id)
@@ -86,16 +85,16 @@ describe('Datasource triggering', () => {
     expect(datasourceResponse.status).toEqual(201)
     const datasourceId = datasourceResponse.body.id
 
-    const dataMetaData = await request(ADAPTER_URL)
+    const dataImportMetadata = await request(ADAPTER_URL)
       .post(`/datasources/${datasourceId}/trigger`)
       .send(null)
-    expect(dataMetaData.status).toEqual(200)
-    expect(dataMetaData.body.id).toBeGreaterThanOrEqual(0)
-    const id = dataMetaData.body.id
-    expect(dataMetaData.body.location).toEqual(`/data/${id}`)
+    expect(dataImportMetadata.status).toEqual(200)
+    expect(dataImportMetadata.body.id).toBeGreaterThanOrEqual(0)
+    const dataImportId = dataImportMetadata.body.id
+    expect(dataImportMetadata.body.location).toEqual(`/datasources/${datasourceId}/imports/${dataImportId}/data`)
 
     const data = await request(ADAPTER_URL)
-      .get(`/data/${id}`)
+      .get(dataImportMetadata.body.location)
       .send()
     expect(data.status).toEqual(200)
     expect(data.body.id).toBe(dynamicDatasourceConfig.protocol.parameters.defaultParameters.id)
@@ -124,14 +123,14 @@ describe('Datasource triggering', () => {
       .send(null)
     expect(dataMetaData.status).toEqual(200)
     expect(dataMetaData.body.id).toBeGreaterThanOrEqual(0)
-    const id = dataMetaData.body.id
-    expect(dataMetaData.body.location).toEqual(`/data/${id}`)
+    const dataImportId = dataMetaData.body.id
+    expect(dataMetaData.body.location).toEqual(`/datasources/${datasourceId}/imports/${dataImportId}/data`)
 
-    const normalData = await request(ADAPTER_URL)
-      .get(`/data/${id}`)
+    const data = await request(ADAPTER_URL)
+      .get(dataMetaData.body.location)
       .send()
-    expect(normalData.status).toEqual(200)
-    expect(normalData.body.id).toEqual('1')
+    expect(data.status).toEqual(200)
+    expect(data.body.id).toEqual('1')
 
     const delResponse = await request(ADAPTER_URL)
       .delete(`/datasources/${datasourceId}`)
@@ -220,7 +219,7 @@ describe('Datasource triggering', () => {
     expect(triggerResponse.status).toEqual(200)
 
     const dataResponse = await request(ADAPTER_URL)
-      .get(`/data/${triggerResponse.body.id}`)
+      .get(triggerResponse.body.location)
     expect(dataResponse.status).toEqual(200)
     expect(dataResponse.body).toEqual({ id: '1' })
   })

adapter/src/main/java/org/jvalue/ods/adapterservice/adapter/Adapter.java

@@ -7,10 +7,8 @@
 import org.jvalue.ods.adapterservice.adapter.model.DataImportResponse;
 import org.jvalue.ods.adapterservice.adapter.model.FormatConfig;
 import org.jvalue.ods.adapterservice.adapter.model.ProtocolConfig;
-import org.springframework.http.HttpStatus;
+import org.jvalue.ods.adapterservice.adapter.model.exceptions.*;
 import org.springframework.stereotype.Service;
-import org.springframework.web.client.RestClientException;
-import org.springframework.web.server.ResponseStatusException;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -25,40 +23,29 @@ public class Adapter {
    *
    * @param config the adapter configuration
    * @return the imported and interpreted data
-   * @throws IllegalArgumentException on errors in the adapter config (e.g. missing parameters, ...)
-   * @throws RestClientException      on response errors when importing the data
+   * @throws ImporterParameterException    on errors in the interpreter config (e.g. missing parameters, ...)
+   * @throws InterpreterParameterException on errors in the interpreter config (e.g. missing parameters, ...)         
+   * @throws IOException                   on response errors when importing the data
    */
-  public DataImportResponse executeJob(AdapterConfig config) throws IllegalArgumentException, RestClientException {
+  public DataImportResponse executeJob(AdapterConfig config) throws ImporterParameterException, InterpreterParameterException, IOException {
     var rawData = this.executeProtocol(config.protocolConfig);
     var result = this.executeFormat(rawData, config.formatConfig);
     return new DataImportResponse(result.toString());
   }
 
-  public DataImportResponse executeRawImport(ProtocolConfig config) {
+  public DataImportResponse executeRawImport(ProtocolConfig config) throws ImporterParameterException {
     var rawData = this.executeProtocol(config);
     return new DataImportResponse(rawData);
   }
 
-  public String executeProtocol(ProtocolConfig config) {
+  public String executeProtocol(ProtocolConfig config) throws ImporterParameterException {
     var importer = config.protocol.getImporter();
-    try {
-      return importer.fetch(config.parameters);
-    } catch (IllegalArgumentException e) {
-      throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid protocol parameters", e);
-    } catch (RestClientException e) {
-      throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to load data: ", e);
-    }
+    return importer.fetch(config.parameters);
   }
 
-  public JsonNode executeFormat(String rawData, FormatConfig config) {
+  public JsonNode executeFormat(String rawData, FormatConfig config) throws InterpreterParameterException, IOException {
     var interpreter = config.format.getInterpreter();
-    try {
-      return interpreter.interpret(rawData, config.parameters);
-    } catch (IOException e) {
-      throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Not able to parse data as format: " + config.format, e);
-    } catch (Exception e) {
-      throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Could not interpret data", e);
-    }
+    return interpreter.interpret(rawData, config.parameters);
   }
 
   public Collection<Importer> getAllProtocols() {

adapter/src/main/java/org/jvalue/ods/adapterservice/adapter/api/rest/v1/AdapterEndpoint.java

@@ -2,12 +2,11 @@
 
 import lombok.AllArgsConstructor;
 import org.jvalue.ods.adapterservice.adapter.Adapter;
-import org.jvalue.ods.adapterservice.adapter.model.AdapterConfig;
-import org.jvalue.ods.adapterservice.adapter.model.DataImportResponse;
-import org.jvalue.ods.adapterservice.adapter.model.ProtocolConfig;
-import org.springframework.http.HttpStatus;
+import org.jvalue.ods.adapterservice.adapter.model.*;
+import org.jvalue.ods.adapterservice.adapter.model.exceptions.*;
 import org.springframework.web.bind.annotation.*;
-import org.springframework.web.server.ResponseStatusException;
+
+import java.io.IOException;
 
 import javax.validation.Valid;
 
@@ -17,24 +16,14 @@ public class AdapterEndpoint {
   private final Adapter adapter;
 
   @PostMapping(Mappings.IMPORT_PATH)
-  public DataImportResponse executeDataImport(@Valid @RequestBody AdapterConfig config) {
-    try {
-      return adapter.executeJob(config);
-    } catch (ResponseStatusException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
-    }
+  public DataImportResponse executeDataImport(@Valid @RequestBody AdapterConfig config)
+      throws ImporterParameterException, InterpreterParameterException, IOException {
+    return adapter.executeJob(config);
   }
 
   @PostMapping(Mappings.RAW_IMPORT_PATH)
-  public DataImportResponse executeRawPreview(@Valid @RequestBody ProtocolConfig config) {
-    try {
-      return adapter.executeRawImport(config);
-    } catch (ResponseStatusException e) {
-      throw e;
-    } catch (Exception e) {
-      throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage());
-    }
+  public DataImportResponse executeRawPreview(@Valid @RequestBody ProtocolConfig config)
+      throws ImporterParameterException {
+    return adapter.executeRawImport(config);
   }
 }

adapter/src/main/java/org/jvalue/ods/adapterservice/adapter/importer/HttpImporter.java

@@ -1,6 +1,8 @@
 package org.jvalue.ods.adapterservice.adapter.importer;
 
 import lombok.AllArgsConstructor;
+
+import org.jvalue.ods.adapterservice.adapter.model.exceptions.ImporterParameterException;
 import org.jvalue.ods.adapterservice.datasource.model.RuntimeParameters;
 import org.springframework.web.client.RestTemplate;
 
@@ -14,10 +16,11 @@
 public class HttpImporter extends Importer {
 
   private final List<ImporterParameterDescription> parameters = List.of(
-    new ImporterParameterDescription("location", "String of the URI for the HTTP call", String.class),
-    new ImporterParameterDescription("encoding", "Encoding of the source. Available encodings: ISO-8859-1, US-ASCII, UTF-8", String.class),
-    new ImporterParameterDescription("defaultParameters", "Default values for open parameters in the URI", false, RuntimeParameters.class)
-  );
+      new ImporterParameterDescription("location", "String of the URI for the HTTP call", String.class),
+      new ImporterParameterDescription("encoding",
+          "Encoding of the source. Available encodings: ISO-8859-1, US-ASCII, UTF-8", String.class),
+      new ImporterParameterDescription("defaultParameters", "Default values for open parameters in the URI", false,
+          RuntimeParameters.class));
   private final RestTemplate restTemplate;
 
   @Override
@@ -31,7 +34,7 @@ public String getDescription() {
   }
 
   @Override
-  protected void validateParameters(Map<String, Object> inputParameters) {
+  protected void validateParameters(Map<String, Object> inputParameters) throws ImporterParameterException {
     super.validateParameters(inputParameters);
 
     String encoding = (String) inputParameters.get("encoding");
@@ -50,12 +53,14 @@ public List<ImporterParameterDescription> getAvailableParameters() {
   }
 
   @Override
-  protected String doFetch(Map<String, Object> parameters) {
-    validateParameters(parameters);
+  protected String doFetch(Map<String, Object> parameters) throws ImporterParameterException {
     String location = parameters.get("location").toString();
-
-    URI uri = URI.create(location);
-    byte[] rawResponse = restTemplate.getForEntity(uri, byte[].class).getBody();
-    return new String(rawResponse, Charset.forName((String) parameters.get("encoding")));
+    try {
+      URI uri = URI.create(location);
+      byte[] rawResponse = restTemplate.getForEntity(uri, byte[].class).getBody();
+      return new String(rawResponse, Charset.forName((String) parameters.get("encoding")));
+    } catch (IllegalArgumentException e) {
+      throw new ImporterParameterException(e.getMessage());
+    }
   }
 }

adapter/src/main/java/org/jvalue/ods/adapterservice/adapter/importer/Importer.java

@@ -6,6 +6,8 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 
+import org.jvalue.ods.adapterservice.adapter.model.exceptions.ImporterParameterException;
+
 public abstract class Importer {
 
   public abstract String getType();
@@ -20,14 +22,14 @@ protected List<ImporterParameterDescription> getRequiredParameters() {
       .filter(ImporterParameterDescription::isRequired).collect(Collectors.toList());
   }
 
-  public final String fetch(Map<String, Object> parameters) {
+  public final String fetch(Map<String, Object> parameters) throws ImporterParameterException {
     validateParameters(parameters);
     return doFetch(parameters);
   }
 
-  protected abstract String doFetch(Map<String, Object> parameters);
+  protected abstract String doFetch(Map<String, Object> parameters) throws ImporterParameterException;
 
-  protected void validateParameters(Map<String, Object> inputParameters) {
+  protected void validateParameters(Map<String, Object> inputParameters) throws ImporterParameterException {
     boolean illegalArguments = false;
     String illegalArgumentsMessage = "";
 
@@ -57,7 +59,7 @@ protected void validateParameters(Map<String, Object> inputParameters) {
       }
     }
     if (illegalArguments) {
-      throw new IllegalArgumentException(illegalArgumentsMessage);
+      throw new ImporterParameterException(illegalArgumentsMessage);
     }
   }