activate audit filter for all v2-pre routes + refactoring (#11205)

refs: #11051

backend/de.metas.business.rest-api-impl/src/main/java/de/metas/rest_api/v2/bpartner/BpartnerRestController.java

@@ -72,7 +72,7 @@
 import static de.metas.common.rest_api.v2.SwaggerDocConstants.NEXT_DOC;
 import static de.metas.common.rest_api.v2.SwaggerDocConstants.SINCE_DOC;
 
-@RequestMapping(value = { MetasfreshRestAPIConstants.ENDPOINT_API_V2 + "/bpartner", "/api/v2/bpartner" })
+@RequestMapping(value = { MetasfreshRestAPIConstants.ENDPOINT_API_V2 + "/bpartner" })
 @RestController
 @Profile(Profiles.PROFILE_App)
 // the spelling "Bpartner" is to avoid swagger from spelling it "b-partner-rest.."

backend/de.metas.business.rest-api-impl/src/main/sql/postgresql/system/50-de.metas.business.rest-api-impl/5590710_sys_gh11051_API_Audit_Adjustments.sql

@@ -0,0 +1,22 @@
+-- 2021-06-01T15:06:11.914Z
+-- I forgot to set the DICTIONARY_ID_COMMENTS System Configurator
+UPDATE AD_Column SET FieldLength=1000000,Updated=TO_TIMESTAMP('2021-06-01 18:06:11','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Column_ID=573790
+;
+
+-- 2021-06-01T15:06:14.063Z
+-- I forgot to set the DICTIONARY_ID_COMMENTS System Configurator
+INSERT INTO t_alter_column values('api_request_audit_log','Logmessage','VARCHAR(1000000)',null,null)
+;
+
+-- 2021-06-01T15:06:33.319Z
+-- I forgot to set the DICTIONARY_ID_COMMENTS System Configurator
+UPDATE AD_Column SET IsIdentifier='Y', IsUpdateable='N',Updated=TO_TIMESTAMP('2021-06-01 18:06:33','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Column_ID=573799
+;
+
+-- 2021-06-01T15:07:01.667Z
+-- I forgot to set the DICTIONARY_ID_COMMENTS System Configurator
+UPDATE AD_Column SET IsIdentifier='Y', SeqNo=10,Updated=TO_TIMESTAMP('2021-06-01 18:07:01','YYYY-MM-DD HH24:MI:SS'),UpdatedBy=100 WHERE AD_Column_ID=573800
+;
+
+update api_audit_config set isinvokerwaitsforresult = 'Y' where api_audit_config_id = 540001
+;

backend/de.metas.cucumber/src/test/java/de/metas/cucumber/stepdefs/RESTUtil.java

@@ -22,6 +22,9 @@
 
 package de.metas.cucumber.stepdefs;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import de.metas.JsonObjectMapperHolder;
+import de.metas.common.rest_api.v2.JsonApiResponse;
 import de.metas.common.rest_api.v2.SyncAdvise;
 import de.metas.common.util.CoalesceUtil;
 import de.metas.common.util.EmptyUtil;
@@ -52,13 +55,14 @@
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 
+import static de.metas.util.web.MetasfreshRestAPIConstants.ENDPOINT_API_V2;
 import static org.assertj.core.api.Assertions.*;
 
 @UtilityClass
 public class RESTUtil
 {
 
-	public String getAuthToken(@NonNull final String userLogin, @NonNull final String roleName) throws IOException
+	public String getAuthToken(@NonNull final String userLogin, @NonNull final String roleName)
 	{
 		final IUserDAO userDAO = Services.get(IUserDAO.class);
 		final IRoleDAO roleDAO = Services.get(IRoleDAO.class);
@@ -130,7 +134,20 @@ public APIResponse performHTTPRequest(final String endpointPath,
 
 		final ByteArrayOutputStream stream = new ByteArrayOutputStream();
 		response.getEntity().writeTo(stream);
-		final String content = stream.toString(StandardCharsets.UTF_8.name());
+		final String content;
+
+		if (endpointPath != null && endpointPath.contains(ENDPOINT_API_V2.substring(1)))
+		{
+			final ObjectMapper objectMapper = JsonObjectMapperHolder.newJsonObjectMapper();
+
+			final JsonApiResponse jsonApiResponse = objectMapper.readValue(stream.toString(StandardCharsets.UTF_8.name()), JsonApiResponse.class);
+
+			content = objectMapper.writeValueAsString(jsonApiResponse.getEndpointResponse());
+		}
+		else
+		{
+			content = stream.toString(StandardCharsets.UTF_8.name());
+		}
 
 		return apiResponseBuilder
 				.content(content)

backend/de.metas.cucumber/src/test/resources/de/metas/cucumber/features/bpartner/createBPartnerV2.feature

@@ -6,7 +6,7 @@ Feature: create or update BPartner v2
     Given the existing user with login 'metasfresh' receives a random a API token for the existing role with name 'WebUI'
 
   Scenario: create a BPartner record
-    When a 'PUT' request with the below payload is sent to the metasfresh REST-API 'api/v2-pre/bpartner/001' and fulfills with '201' status code
+    When a 'PUT' request with the below payload is sent to the metasfresh REST-API 'api/v2/bpartner/001' and fulfills with '201' status code
     """
 {
    "requestItems":[
@@ -102,7 +102,7 @@ Feature: create or update BPartner v2
       | ext-ALBERTA-001    | ext-ALBERTA-c22   | test_name_c22 | null       | test_fax | c22  | true                    |
 
   Scenario: Update a BPartner record
-    When a 'PUT' request with the below payload is sent to the metasfresh REST-API 'api/v2-pre/bpartner/001' and fulfills with '201' status code
+    When a 'PUT' request with the below payload is sent to the metasfresh REST-API 'api/v2/bpartner/001' and fulfills with '201' status code
     """
 {
    "requestItems":[
@@ -134,7 +134,7 @@ Feature: create or update BPartner v2
       | ext-ALBERTA-001    | test_code_updated | test_name_updated | test_company    | null         | null      | de           | url_updated | test-group | null      |
 
   Scenario: Update a BPartner contact record
-    When a 'PUT' request with the below payload is sent to the metasfresh REST-API 'api/v2-pre/bpartner/001' and fulfills with '201' status code
+    When a 'PUT' request with the below payload is sent to the metasfresh REST-API 'api/v2/bpartner/001' and fulfills with '201' status code
     """
 {
    "requestItems":[
@@ -169,7 +169,7 @@ Feature: create or update BPartner v2
       | ext-ALBERTA-001    | ext-ALBERTA-c11   | test_name_c11_updated | test_email_updated | fax_updated | c11  | true                    |
 
   Scenario: Update a BPartner contact record and Create another contact record
-    When a 'PUT' request with the below payload is sent to the metasfresh REST-API 'api/v2-pre/bpartner/001' and fulfills with '201' status code
+    When a 'PUT' request with the below payload is sent to the metasfresh REST-API 'api/v2/bpartner/001' and fulfills with '201' status code
     """
 {
    "requestItems":[

backend/de.metas.cucumber/src/test/resources/de/metas/cucumber/features/createPurchaseCandidateV2.feature

@@ -22,12 +22,12 @@ Feature: create or update Purchase Candidate
 
   Scenario:  The purchase candidate request is set in context, enqueued and validated
     And the purchase candidate request is set in context
-    When the metasfresh REST-API endpoint path 'api/v2-pre/order/purchase/createCandidates' receives a 'POST' request with the payload from context and responds with '200' status code
+    When the metasfresh REST-API endpoint path 'api/v2/order/purchase/createCandidates' receives a 'POST' request with the payload from context and responds with '200' status code
     Then verify if data is persisted correctly for purchase candidate
 
     Given the purchase candidate enqueue-status request is set in context
-    Then the metasfresh REST-API endpoint path 'api/v2-pre/order/purchase/enqueueForOrdering' receives a 'POST' request with the payload from context and responds with '202' status code
+    Then the metasfresh REST-API endpoint path 'api/v2/order/purchase/enqueueForOrdering' receives a 'POST' request with the payload from context and responds with '202' status code
 
     Given the purchase candidate enqueue-status request is set in context
-    When the metasfresh REST-API endpoint path 'api/v2-pre/order/purchase/status' receives a 'PUT' request with the payload from context and responds with '200' status code
+    When the metasfresh REST-API endpoint path 'api/v2/order/purchase/status' receives a 'PUT' request with the payload from context and responds with '200' status code
     Then verify purchase candidate status

backend/de.metas.cucumber/src/test/resources/de/metas/cucumber/features/upsertPrices.feature

@@ -15,15 +15,15 @@ Feature: Create or update using prices API
       | Identifier  | OrgCode | M_PriceList_ID | OPT.Description                | ValidFrom            | OPT.IsActive |
       | ext-Other-3 | 001     | 2              | price_list_version_description | 2018-11-12T00:00:00Z | true         |
     And we create a JsonRequestPriceListVersionUpsert, set syncAdvise to 'CREATE_OR_MERGE' and store request payload it in the test context
-    And the metasfresh REST-API endpoint path '/api/v2-pre/prices/priceListVersions' receives a 'PUT' request with the payload from context and responds with '200' status code
+    And the metasfresh REST-API endpoint path 'api/v2/prices/priceListVersions' receives a 'PUT' request with the payload from context and responds with '200' status code
     Then price list version is persisted correctly
 
   Scenario: Update price list version
     When the user adds price list version data
       | Identifier  | OrgCode | M_PriceList_ID | OPT.Description            | ValidFrom            | OPT.IsActive |
       | ext-Other-3 | 001     | 2              | price_list_version_updated | 2017-01-17T00:00:00Z | false        |
     And we create a JsonRequestPriceListVersionUpsert, set syncAdvise to 'CREATE_OR_MERGE' and store request payload it in the test context
-    When the metasfresh REST-API endpoint path '/api/v2-pre/prices/priceListVersions' receives a 'PUT' request with the payload from context and responds with '200' status code
+    When the metasfresh REST-API endpoint path 'api/v2/prices/priceListVersions' receives a 'PUT' request with the payload from context and responds with '200' status code
     Then price list version is persisted correctly
 
   Scenario: Create product prices by price list version identifier
@@ -32,13 +32,13 @@ Feature: Create or update using prices API
       | Identifier  | OrgCode | M_Product_ID | PriceStd | OPT.PriceLimit | OPT.PriceList | SeqNo | OPT.IsActive | TaxCategory.InternalName |
       | ext-Other-4 | 001     | 123          | 10       | 20             | 30            | 10    | false        | NORMAL                   |
     And we create a JsonRequestProductPriceUpsert, set syncAdvise to 'CREATE_OR_MERGE' and store request payload it in the test context
-    When the metasfresh REST-API endpoint path '/api/v2-pre/prices/priceListVersions/ext-Other-3/productPrices' receives a 'PUT' request with the payload from context and responds with '200' status code
+    When the metasfresh REST-API endpoint path 'api/v2/prices/priceListVersions/ext-Other-3/productPrices' receives a 'PUT' request with the payload from context and responds with '200' status code
     Then product price is persisted correctly
 
   Scenario: Update product prices by price list version identifier
     When the user adds product prices data
       | Identifier  | OrgCode | M_Product_ID | PriceStd | OPT.PriceLimit | OPT.PriceList | SeqNo | OPT.IsActive | TaxCategory.InternalName |
       | ext-Other-4 | 001     | 123          | 22       | 20             | 30            | 10    | true         | NORMAL                   |
     And we create a JsonRequestProductPriceUpsert, set syncAdvise to 'CREATE_OR_MERGE' and store request payload it in the test context
-    When the metasfresh REST-API endpoint path '/api/v2-pre/prices/priceListVersions/ext-Other-3/productPrices' receives a 'PUT' request with the payload from context and responds with '200' status code
+    When the metasfresh REST-API endpoint path 'api/v2/prices/priceListVersions/ext-Other-3/productPrices' receives a 'PUT' request with the payload from context and responds with '200' status code
     Then product price is persisted correctly

backend/de.metas.cucumber/src/test/resources/de/metas/cucumber/features/upsertProduct.feature

@@ -27,7 +27,7 @@ Feature:product create/update using metasfresh api
 
     And we create a JsonRequestProductUpsert, set syncAdvise to 'CREATE_OR_MERGE', add the JsonRequestProductUpsertItems and store it in the test context
 
-    When the metasfresh REST-API endpoint path '/api/v2-pre/products/001' receives a 'PUT' request with the payload from context and responds with '200' status code
+    When the metasfresh REST-API endpoint path 'api/v2/products/001' receives a 'PUT' request with the payload from context and responds with '200' status code
     Then verify if data is persisted correctly for each product
     And verify if data is persisted correctly for type 'product' and external reference 'ext-ALBERTA-345'
 

backend/de.metas.util.web/src/main/java/de/metas/util/web/MetasfreshRestAPIConstants.java

@@ -32,7 +32,7 @@ public class MetasfreshRestAPIConstants
 	 */
 	public static final String URL_PATTERN_API = "/api/*";
 
-	public static final String URL_PATTERN_API_V2 = "/api/v2/*"; //FIXME: temporary
+	public static final String URL_PATTERN_API_V2 = "/api/v2/*";
 
 	/**
 	 * @deprecated please invoke endpoints with either {@link #ENDPOINT_API_V1} or {@link #ENDPOINT_API_V2}.
@@ -44,5 +44,5 @@ public class MetasfreshRestAPIConstants
 	/**
 	 * going to be renamed to {@code v2} soon, after some initial breaking changes were made
 	 */
-	public static final String ENDPOINT_API_V2 = "/api/v2-pre";
+	public static final String ENDPOINT_API_V2 = "/api/v2";
 }

backend/de.metas.util.web/src/main/java/de/metas/util/web/audit/ApiAuditService.java

@@ -41,6 +41,7 @@
 import de.metas.audit.response.ApiResponseAudit;
 import de.metas.audit.response.ApiResponseAuditRepository;
 import de.metas.common.rest_api.common.JsonMetasfreshId;
+import de.metas.common.rest_api.v2.JsonApiResponse;
 import de.metas.i18n.AdMessageKey;
 import de.metas.notification.INotificationBL;
 import de.metas.notification.UserNotificationRequest;
@@ -73,6 +74,7 @@
 import java.net.URI;
 import java.time.Instant;
 import java.util.Comparator;
+import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -130,11 +132,11 @@ public Optional<ApiRequestAuditId> extractApiRequestAuditId(@NonNull final HttpS
 				.map(requestId -> ApiRequestAuditId.ofRepoId(NumberUtils.asInt(requestId, -1)));
 	}
 
-	public ApiAuditLoggable createLogger(@NonNull final ApiRequestAuditId apiRequestAuditId)
+	public ApiAuditLoggable createLogger(@NonNull final ApiRequestAuditId apiRequestAuditId, @NonNull final UserId userId)
 	{
 		return ApiAuditLoggable.builder()
 				.clientId(Env.getClientId())
-				.userId(Env.getLoggedUserId())
+				.userId(userId)
 				.apiRequestAuditId(apiRequestAuditId)
 				.apiAuditRequestLogDAO(apiAuditRequestLogDAO)
 				.bufferSize(100)
@@ -170,7 +172,7 @@ public void processHttpCall(
 
 		final ApiRequestAudit requestAudit = logRequest(httpRequest, apiAuditConfig.getApiAuditConfigId(), orgId);
 
-		final ApiAuditLoggable apiAuditLoggable = createLogger(requestAudit.getIdNotNull());
+		final ApiAuditLoggable apiAuditLoggable = createLogger(requestAudit.getIdNotNull(), requestAudit.getUserId());
 
 		try (final IAutoCloseable loggableRestorer = Loggables.temporarySetLoggable(apiAuditLoggable))
 		{
@@ -245,9 +247,10 @@ public ApiResponse executeHttpCall(@NonNull final ApiRequestAudit apiRequestAudi
 		return bodySpec.exchangeToMono(cr -> cr
 				.bodyToMono(Object.class)
 
-				.defaultIfEmpty(ApiResponse.of(cr.rawStatusCode(), cr.headers().asHttpHeaders(), null))
+				.map(body -> ApiResponse.of(cr.rawStatusCode(), cr.headers().asHttpHeaders(), body))
+
+				.defaultIfEmpty(ApiResponse.of(cr.rawStatusCode(), cr.headers().asHttpHeaders(), null)))
 
-				.map(body -> ApiResponse.of(cr.rawStatusCode(), cr.headers().asHttpHeaders(), body)))
 				.block();
 	}
 
@@ -428,8 +431,7 @@ private void handleSuccessfulResponse(
 
 		if (actualAPIResponse.getHttpHeaders() != null)
 		{
-			actualAPIResponse.getHttpHeaders()
-					.forEach((key, values) -> values.forEach(value -> httpServletResponse.addHeader(key, value)));
+			forwardResponseHttpHeaders(actualAPIResponse.getHttpHeaders(), httpServletResponse);
 		}
 
 		httpServletResponse.resetBuffer();
@@ -447,6 +449,21 @@ private ApiResponse getGenericNoWaitResponse()
 		return ApiResponse.of(HttpStatus.ACCEPTED.value(), httpHeaders, null);
 	}
 
+	private void forwardResponseHttpHeaders(@NonNull final HttpHeaders httpHeaders, @NonNull final HttpServletResponse servletResponse)
+	{
+		httpHeaders.keySet()
+				.stream()
+				.filter(key -> !key.equals(HttpHeaders.CONNECTION))
+				.filter(key -> !key.equals(HttpHeaders.CONTENT_LENGTH))
+				.forEach(key -> {
+					final List<String> values = httpHeaders.get(key);
+					if (values != null)
+					{
+						values.forEach(value -> servletResponse.addHeader(key, value));
+					}
+				});
+	}
+
 	@Value
 	@Builder
 	private static class FutureCompletionContext

backend/de.metas.util.web/src/main/java/de/metas/util/web/audit/ApiRequestReplayService.java

@@ -49,7 +49,7 @@ public void replayApiRequests(@NonNull final ImmutableList<ApiRequestAudit> apiR
 
 	private void replayActionNoFailing(@NonNull final ApiRequestAudit apiRequestAudit)
 	{
-		final ApiAuditLoggable loggable = apiAuditService.createLogger(apiRequestAudit.getIdNotNull());
+		final ApiAuditLoggable loggable = apiAuditService.createLogger(apiRequestAudit.getIdNotNull(), apiRequestAudit.getUserId());
 
 		try (final IAutoCloseable loggableRestorer = Loggables.temporarySetLoggable(loggable))
 		{

backend/de.metas.util.web/src/main/java/de/metas/util/web/filter/ApiAuditFilter.java

@@ -28,6 +28,7 @@
 import de.metas.util.Loggables;
 import de.metas.util.web.audit.ApiAuditService;
 import org.adempiere.util.lang.IAutoCloseable;
+import org.compiere.util.Env;
 
 import javax.servlet.Filter;
 import javax.servlet.FilterChain;
@@ -67,7 +68,7 @@ public void doFilter(final ServletRequest request, final ServletResponse respons
 			// dev-note: this means the request was already filtered once
 			if (requestAuditIdOpt.isPresent())
 			{
-				final ApiAuditLoggable apiAuditLoggable = apiAuditService.createLogger(requestAuditIdOpt.get());
+				final ApiAuditLoggable apiAuditLoggable = apiAuditService.createLogger(requestAuditIdOpt.get(), Env.getLoggedUserId());
 
 				try (final IAutoCloseable loggableRestorer = Loggables.temporarySetLoggable(apiAuditLoggable))
 				{

backend/de.metas.util.web/src/main/java/de/metas/util/web/filter/FilterRegistrationConfig.java

@@ -33,6 +33,8 @@
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
 
+import static de.metas.util.web.MetasfreshRestAPIConstants.URL_PATTERN_API_V2;
+
 @Profile(Profiles.PROFILE_App)
 @Configuration
 public class FilterRegistrationConfig
@@ -43,7 +45,7 @@ public FilterRegistrationBean<ApiAuditFilter> apiAuditFilter(@NonNull final ApiA
 		final FilterRegistrationBean<ApiAuditFilter> registrationBean = new FilterRegistrationBean<>();
 
 		registrationBean.setFilter(new ApiAuditFilter(apiAuditService));
-		registrationBean.addUrlPatterns("/api/v2/*"); //FIXME temporary
+		registrationBean.addUrlPatterns(URL_PATTERN_API_V2);
 		registrationBean.setOrder(2);
 		return registrationBean;
 	}

backend/de.metas.util.web/src/main/java/de/metas/util/web/audit/JsonApiResponse.java → misc/de-metas-common/de-metas-common-rest_api/src/main/java/de/metas/common/rest_api/v2/JsonApiResponse.java

@@ -20,10 +20,11 @@
  * #L%
  */
 
-package de.metas.util.web.audit;
+package de.metas.common.rest_api.v2;
 
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import de.metas.common.rest_api.common.JsonMetasfreshId;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Builder;
@@ -33,6 +34,7 @@
 
 @Value
 @Builder
+@JsonDeserialize(builder = JsonApiResponse.JsonApiResponseBuilder.class)
 public class JsonApiResponse
 {
 	@ApiModelProperty("The `API_Request_Audit_ID` of the request created by the invoker. Might be `null` if metasfresh was configured to not create an audit log at all.")