From 35d33bfa1361d601f741f1c22baa14923790a9e3 Mon Sep 17 00:00:00 2001 From: Raido Kaju Date: Tue, 18 Jun 2024 09:32:25 +0300 Subject: [PATCH] XRDCAT-12: Improve Company and Organization handling processes (#28) --- doc/xroad_catalog_installation_guide.md | 23 +- .../collector/XRoadCatalogCollector.java | 33 +- .../configuration/TaskPoolConfiguration.java | 42 +-- .../collector/tasks/BaseFetchTask.java | 2 +- .../collector/tasks/FetchCompaniesTask.java | 36 +- .../tasks/FetchOrganizationsTask.java | 51 +-- .../collector/tasks/ListClientsTask.java | 37 +-- .../collector/tasks/ListMethodsTask.java | 2 +- .../collector/tasks/UpdateExternalsTask.java | 85 +++++ .../collector/util/ClientListUtil.java | 49 +-- .../collector/util/OrganizationUtil.java | 314 +++++------------- .../tasks/FetchCompaniesTaskTest.java | 60 ++-- .../tasks/FetchOrganizationTaskTest.java | 63 ++-- .../collector/tasks/ListClientsTaskTest.java | 55 +-- .../mock/companies/getCompanies.json | 39 --- .../catalog/persistence/CatalogService.java | 15 +- .../persistence/CatalogServiceImpl.java | 24 +- .../repository/MemberRepository.java | 30 +- 18 files changed, 425 insertions(+), 535 deletions(-) create mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/UpdateExternalsTask.java delete mode 100644 xroad-catalog-collector/src/test/resources/mock/companies/getCompanies.json diff --git a/doc/xroad_catalog_installation_guide.md b/doc/xroad_catalog_installation_guide.md index 2abe8d77..167a09aa 100644 --- a/doc/xroad_catalog_installation_guide.md +++ b/doc/xroad_catalog_installation_guide.md @@ -1,5 +1,5 @@ # X-Road Catalog Installation Guide -Version: 1.3.1 +Version: 1.3.2 Doc. ID: IG-XRDCAT --- @@ -12,6 +12,7 @@ Doc. ID: IG-XRDCAT | 09.09.2023 | 1.2.0 | Remove instructions to install the `xroad-conflient` module manually | Petteri Kivimäki | | 24.09.2023 | 1.3.0 | Add instructions to disable the automatic backup job run by the `xroad-conflient` module | Petteri Kivimäki | | 10.06.2024 | 1.3.1 | Add information about default values for configurable properties | Raido Kaju | +| 14.06.2024 | 1.3.2 | Update information about company and organization task properties | Raido Kaju | ## Table of Contents @@ -189,16 +190,16 @@ Optional parameters which can be configured in the same file are described below When using the `xroad-catalog-collector` module with the `FI` profile, the following additional optional parameters are in effect: -| Parameter | Defaults | Description | -|--------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `xroad-catalog.max-organizations-per-request` | 100 | A parameter for setting the maximum amount of organizations that can be fetched from the organizations API in one request, e.g. value `100` means `100 organizations`. | -| `xroad-catalog.fetch-companies-limit` | 1000 | A parameter for setting the maximum amount of companies that can be fetched from the companies API, e.g. value `1000` means `1000 companies`. | -| `xroad-catalog.fetch-organizations-limit` | 2000 | A parameter for setting the maximum amount of organizations that can be fetched from the organizations API, e.g. value `2000` means `2000 organizations`. | -| `xroad-catalog.fetch-companies-run-unlimited` | false | A parameter for setting whether the X-Road Catalog Collector should try to fetch data from the companies API continuously during a day or only between certain hours, e.g. value `true` means `continously`. | -| `xroad-catalog.fetch-companies-time-after-hour` | 3 | A parameter for setting the start of time interval during which the X-Road Catalog Collector should try to fetch data from the companies API continuously (this parameter will be ignored if the parameter `xroad-catalog.fetch-companies-run-unlimited` is set to `true`), e.g. value `18` means starting from `18:00`. | -| `xroad-catalog.fetch-companies-time-before-hour` | 4 | A parameter for setting the end of time interval during which the X-Road Catalog Collector should try to fetch data from the companies API continuously (this parameter will be ignored if the parameter `xroad-catalog.fetch-companies-run-unlimited` is set to `true`), e.g. value `23` means ending at `23:00`. | -| `xroad-catalog.fetch-organizations-pool-size` | 10 | A parameter for setting the amount of virtual threads in the pool for fetching organizations from the organizations API, e.g. value `10` means `10 virtual threads`. | -| `xroad-catalog.fetch-companies-pool-size` | 10 | A parameter for setting the amount of virtual threads in the pool for fetching companies from the companies API, e.g. value `10` means `10 virtual threads`. | +| Parameter | Defaults | Description | +|--------------------------------------------------|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `xroad-catalog.fetch-external-run-unlimited` | false | A parameter for setting whether the X-Road Catalog Collector should try to fetch data from the external API continuously during a day or only between certain hours, e.g. value `true` means `continously`. | +| `xroad-catalog.fetch-external-time-after-hour` | 3 | A parameter for setting the start of time interval during which the X-Road Catalog Collector should try to fetch data from the companies API continuously (this parameter will be ignored if the parameter `xroad-catalog.fetch-companies-run-unlimited` is set to `true`), e.g. value `18` means starting from `18:00`. | +| `xroad-catalog.fetch-external-time-before-hour` | 4 | A parameter for setting the end of time interval during which the X-Road Catalog Collector should try to fetch data from the companies API continuously (this parameter will be ignored if the parameter `xroad-catalog.fetch-companies-run-unlimited` is set to `true`), e.g. value `23` means ending at `23:00`. | +| `xroad-catalog.fetch-external-interval-min` | 20 | A parameter for setting the amount of time in minutes after which the X-Road Catalog Collector should start re-fetching data from the external API, e.g. value `20` means `every 20 minutes`. This works together with the following two parameters to determine how often data is checked for staleness and updated in a batch. | +| `xroad-catalog.fetch-external-limit` | 500 | A parameter for setting the maximum amount of Members that should be fetched per external API in one run, e.g. value `500` means `500 members`. In the current implementation the example value would fetch `500` members information from both the `company` and `organization` API. | +| `xroad-catalog.fetch-external-update-after-days` | 7 | A parameter for setting the amount of days after which the X-Road Catalog Collector should consider Company and Organization data stale and try to fetch data from the external API again, e.g. value `7` means `after 7 days`. | +| `xroad-catalog.fetch-organizations-pool-size` | 10 | A parameter for setting the amount of virtual threads in the pool for fetching organizations from the organizations API, e.g. value `10` means `10 virtual threads`. This controls how many parallel requests will hit the organizations API. | +| `xroad-catalog.fetch-companies-pool-size` | 10 | A parameter for setting the amount of virtual threads in the pool for fetching companies from the companies API, e.g. value `10` means `10 virtual threads`. This controls how many parallel requests will hit the companies API. | In addition, update the `xroad-catalog.shared-params-file` property value in `/etc/xroad/xroad-catalog/lister-production.properties`. The value must point to the `/etc/xroad/globalconf//shared-params.xml` X-Road global configuration file: diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/XRoadCatalogCollector.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/XRoadCatalogCollector.java index 32f7dd7c..ce8eae83 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/XRoadCatalogCollector.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/XRoadCatalogCollector.java @@ -35,6 +35,7 @@ import fi.vrk.xroad.catalog.collector.tasks.FetchWsdlsTask; import fi.vrk.xroad.catalog.collector.tasks.ListClientsTask; import fi.vrk.xroad.catalog.collector.tasks.ListMethodsTask; +import fi.vrk.xroad.catalog.collector.tasks.UpdateExternalsTask; import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; import fi.vrk.xroad.catalog.collector.wsimport.ClientType; import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; @@ -65,47 +66,57 @@ public static void main(String[] args) throws MalformedURLException, URISyntaxEx } } + final TaskPoolConfiguration taskPoolConfiguration = context.getBean(TaskPoolConfiguration.class); + final boolean isFIProfile = Arrays.stream(env.getActiveProfiles()) .anyMatch(str -> str.equalsIgnoreCase(FI_PROFILE)); final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); final BlockingQueue listMethodsQueue = new LinkedBlockingQueue<>(); - final BlockingQueue fetchCompaniesQueue = isFIProfile ? new LinkedBlockingQueue<>() : null; - final BlockingQueue fetchOrganizationsQueue = isFIProfile ? new LinkedBlockingQueue<>() : null; + final BlockingQueue fetchCompaniesQueue = isFIProfile ? new LinkedBlockingQueue<>() : null; + final BlockingQueue fetchOrganizationsQueue = isFIProfile ? new LinkedBlockingQueue<>() : null; final BlockingQueue fetchWsdlsQueue = new LinkedBlockingQueue<>(); final BlockingQueue fetchRestQueue = new LinkedBlockingQueue<>(); final BlockingQueue fetchOpenApiQueue = new LinkedBlockingQueue<>(); if (isFIProfile) { log.info("FI profile detected, starting up organizations and companies fetchers"); + final FetchCompaniesTask fetchCompaniesTask = new FetchCompaniesTask(context, fetchCompaniesQueue); - Thread.ofVirtual().start(fetchCompaniesTask::run); + Thread.ofVirtual().start(fetchCompaniesTask); final FetchOrganizationsTask fetchOrganizationsTask = new FetchOrganizationsTask(context, fetchOrganizationsQueue); - Thread.ofVirtual().start(fetchOrganizationsTask::run); + Thread.ofVirtual().start(fetchOrganizationsTask); + + final UpdateExternalsTask updateExternalsTask = new UpdateExternalsTask(context, fetchCompaniesQueue, + fetchOrganizationsQueue); + long externalInterval = taskPoolConfiguration.getFetchExternalInterval(); + log.info("Starting up external sources updater with interval of {}", externalInterval); + + scheduler.scheduleWithFixedDelay(updateExternalsTask, 0, externalInterval, TimeUnit.MINUTES); } final FetchWsdlsTask fetchWsdlsTask = new FetchWsdlsTask(context, fetchWsdlsQueue); - Thread.ofVirtual().start(fetchWsdlsTask::run); + Thread.ofVirtual().start(fetchWsdlsTask); final FetchRestTask fetchRestTask = new FetchRestTask(context, fetchRestQueue); - Thread.ofVirtual().start(fetchRestTask::run); + Thread.ofVirtual().start(fetchRestTask); final FetchOpenApiTask fetchOpenApiTask = new FetchOpenApiTask(context, fetchOpenApiQueue); - Thread.ofVirtual().start(fetchOpenApiTask::run); + Thread.ofVirtual().start(fetchOpenApiTask); final ListMethodsTask listMethodsTask = new ListMethodsTask(context, listMethodsQueue, fetchWsdlsQueue, - fetchRestQueue, - fetchOpenApiQueue); - Thread.ofVirtual().start(listMethodsTask::run); + fetchRestQueue, fetchOpenApiQueue); + Thread.ofVirtual().start(listMethodsTask); // The ListClientsTask is the main task that starts the whole process and // gathers information that the other tasks will react on to do work final ListClientsTask listClientsTask = new ListClientsTask(context, listMethodsQueue, fetchCompaniesQueue, fetchOrganizationsQueue); - long collectorInterval = context.getBean(TaskPoolConfiguration.class).getCollectorInterval(); + + long collectorInterval = taskPoolConfiguration.getCollectorInterval(); log.info("Starting up catalog collector with collector interval of {}", collectorInterval); scheduler.scheduleWithFixedDelay(listClientsTask::run, 0, collectorInterval, TimeUnit.MINUTES); diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/TaskPoolConfiguration.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/TaskPoolConfiguration.java index 6e13bd23..dc7e3b40 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/TaskPoolConfiguration.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/TaskPoolConfiguration.java @@ -74,23 +74,31 @@ public class TaskPoolConfiguration { @Value("${xroad-catalog.fetch-companies-url}") private String fetchCompaniesUrl; - @Value("${xroad-catalog.max-organizations-per-request:100}") - private int maxOrganizationsPerRequest; + @Value("${xroad-catalog.fetch-external-limit:500}") + private int fetchExternalLimit; - @Value("${xroad-catalog.fetch-companies-limit:1000}") - private int fetchCompaniesLimit; + @Value("${xroad-catalog.fetch-external-update-after-days:7}") + private int fetchExternalUpdateAfterDays; - @Value("${xroad-catalog.fetch-organizations-limit:2000}") - private int fetchOrganizationsLimit; + @Value("${xroad-catalog.fetch-external-interval-min:20}") + private long fetchExternalInterval; - @Value("${xroad-catalog.fetch-companies-run-unlimited:false}") - private boolean fetchCompaniesRunUnlimited; + @Value("${xroad-catalog.fetch-external-run-unlimited:false}") + private boolean fetchExternalRunUnlimited; - @Value("${xroad-catalog.fetch-companies-time-after-hour:3}") - private int fetchCompaniesTimeAfterHour; + @Value("${xroad-catalog.fetch-external-time-after-hour:3}") + private int fetchExternalTimeAfterHour; - @Value("${xroad-catalog.fetch-companies-time-before-hour:4}") - private int fetchCompaniesTimeBeforeHour; + @Value("${xroad-catalog.fetch-external-time-before-hour:4}") + private int fetchExternalTimeBeforeHour; + + @Value("${xroad-catalog.fetch-organizations-pool-size:10}") + private int fetchOrganizationsPoolSize; + + @Value("${xroad-catalog.fetch-companies-pool-size:10}") + private int fetchCompaniesPoolSize; + + // Parameters handling database log storage @Value("${xroad-catalog.flush-log-time-after-hour:3}") private int flushLogTimeAfterHour; @@ -101,6 +109,8 @@ public class TaskPoolConfiguration { @Value("${xroad-catalog.error-log-length-in-days:90}") private int errorLogLengthInDays; + // Parameters controlling how often data is collected from the X-Road instance + @Value("${xroad-catalog.fetch-run-unlimited:false}") private boolean fetchRunUnlimited; @@ -110,7 +120,7 @@ public class TaskPoolConfiguration { @Value("${xroad-catalog.fetch-time-before-hour:4}") private int fetchTimeBeforeHour; - // Collector internal parameters + // Collector internal pool parameters @Value("${xroad-catalog.collector-interval-min:20}") private long collectorInterval; @@ -127,10 +137,4 @@ public class TaskPoolConfiguration { @Value("${xroad-catalog.fetch-rest-pool-size:10}") private int fetchRestPoolSize; - @Value("${xroad-catalog.fetch-organizations-pool-size:10}") - private int fetchOrganizationsPoolSize; - - @Value("${xroad-catalog.fetch-companies-pool-size:10}") - private int fetchCompaniesPoolSize; - } diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/BaseFetchTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/BaseFetchTask.java index 047338bc..957cb141 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/BaseFetchTask.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/BaseFetchTask.java @@ -39,7 +39,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public abstract class BaseFetchTask { +public abstract class BaseFetchTask implements Runnable { protected final CatalogService catalogService; diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTask.java index 4e594141..14cd31bc 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTask.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTask.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Semaphore; @@ -37,7 +38,6 @@ import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; import fi.vrk.xroad.catalog.collector.util.OrganizationUtil; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; import fi.vrk.xroad.catalog.persistence.CatalogService; import fi.vrk.xroad.catalog.persistence.CompanyService; import fi.vrk.xroad.catalog.persistence.entity.BusinessAddress; @@ -55,24 +55,22 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class FetchCompaniesTask { +public class FetchCompaniesTask implements Runnable { private final String fetchCompaniesUrl; - private final Integer fetchCompaniesLimit; - private final CatalogService catalogService; private final CompanyService companyService; - private final BlockingQueue fetchCompaniesQueue; + private final BlockingQueue fetchCompaniesQueue; private final TaskPoolConfiguration taskPoolConfiguration; private final Semaphore semaphore; public FetchCompaniesTask(final ApplicationContext applicationContext, - final BlockingQueue fetchCompaniesQueue) { + final BlockingQueue fetchCompaniesQueue) { this.catalogService = applicationContext.getBean(CatalogService.class); this.companyService = applicationContext.getBean(CompanyService.class); @@ -80,7 +78,6 @@ public FetchCompaniesTask(final ApplicationContext applicationContext, this.taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); this.fetchCompaniesUrl = taskPoolConfiguration.getFetchCompaniesUrl(); - this.fetchCompaniesLimit = taskPoolConfiguration.getFetchCompaniesLimit(); this.semaphore = new Semaphore(taskPoolConfiguration.getFetchCompaniesPoolSize()); @@ -93,9 +90,9 @@ public void run() { log.debug("Waiting for data ... "); // take() blocks until an element becomes available or it gets interrupted - ClientType client = fetchCompaniesQueue.take(); + String businessId = fetchCompaniesQueue.take(); semaphore.acquire(); - Thread.ofVirtual().start(() -> fetchCompaniesForCLient(client)); + Thread.ofVirtual().start(() -> fetchCompanyData(businessId)); } } catch (InterruptedException e) { log.warn("Interrupted while waiting for data, stopping {}", getClass().getSimpleName(), e); @@ -103,23 +100,15 @@ public void run() { } } - protected void fetchCompaniesForCLient(final ClientType client) { + protected void fetchCompanyData(final String businessId) { try { - JSONObject companiesJson = OrganizationUtil.getCompanies(client, fetchCompaniesLimit, fetchCompaniesUrl, + log.info("Fetching company information for member {}", businessId); + + Optional company = OrganizationUtil.getCompany(fetchCompaniesUrl, businessId, catalogService); - JSONArray companiesArray = companiesJson.optJSONArray("results"); - int numberOfCompanies = companiesArray.length(); - log.info("Fetching data for {} companies", numberOfCompanies); - companiesArray.forEach(item -> { - JSONObject company = (JSONObject) item; - String businessCode = company.optString("businessId"); - JSONObject companyJson = OrganizationUtil.getCompany(client, fetchCompaniesUrl, businessCode, - catalogService); - saveData(companyJson.optJSONArray("results")); - }); - log.info("Successfully saved data for {} companies", numberOfCompanies); + company.ifPresent(companyJson -> saveData(companyJson.optJSONArray("results"))); } catch (Exception e) { - log.error("Error while fetching companies for client {}", client, e); + log.error("Error while fetching company information for member {}", businessId, e); } finally { semaphore.release(); } @@ -139,6 +128,7 @@ private void saveData(JSONArray data) { saveLiquidations(data.optJSONObject(i).optJSONArray("liquidations"), savedCompany); saveRegisteredEntries(data.optJSONObject(i).optJSONArray("registeredEntries"), savedCompany); saveRegisteredOffices(data.optJSONObject(i).optJSONArray("registeredOffices"), savedCompany); + log.info("Company information saved for member {}", savedCompany.getBusinessId()); } } diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationsTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationsTask.java index b2a9cbb7..a0e61197 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationsTask.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationsTask.java @@ -26,11 +26,10 @@ */ package fi.vrk.xroad.catalog.collector.tasks; -import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Semaphore; -import java.util.concurrent.atomic.AtomicInteger; import org.json.JSONArray; import org.json.JSONObject; @@ -38,7 +37,6 @@ import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; import fi.vrk.xroad.catalog.collector.util.OrganizationUtil; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; import fi.vrk.xroad.catalog.persistence.CatalogService; import fi.vrk.xroad.catalog.persistence.OrganizationService; import fi.vrk.xroad.catalog.persistence.entity.Address; @@ -63,26 +61,23 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class FetchOrganizationsTask { +public class FetchOrganizationsTask implements Runnable { private String fetchOrganizationsUrl; - private Integer maxOrganizationsPerRequest; - - private Integer fetchOrganizationsLimit; - private final CatalogService catalogService; private final OrganizationService organizationService; - private final BlockingQueue fetchOrganizationsQueue; + private final BlockingQueue fetchOrganizationsQueue; private final TaskPoolConfiguration taskPoolConfiguration; private final Semaphore semaphore; public FetchOrganizationsTask(final ApplicationContext applicationContext, - final BlockingQueue fetchOrganizationsQueue) { + final BlockingQueue fetchOrganizationsQueue) { + this.catalogService = applicationContext.getBean(CatalogService.class); this.organizationService = applicationContext.getBean(OrganizationService.class); @@ -90,8 +85,6 @@ public FetchOrganizationsTask(final ApplicationContext applicationContext, this.taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); this.fetchOrganizationsUrl = taskPoolConfiguration.getFetchOrganizationsUrl(); - this.maxOrganizationsPerRequest = taskPoolConfiguration.getMaxOrganizationsPerRequest(); - this.fetchOrganizationsLimit = taskPoolConfiguration.getFetchOrganizationsLimit(); this.semaphore = new Semaphore(taskPoolConfiguration.getFetchOrganizationsPoolSize()); @@ -104,9 +97,9 @@ public void run() { log.debug("Waiting for data ... "); // take() blocks until an element becomes available or it gets interrupted - ClientType client = fetchOrganizationsQueue.take(); + String businessId = fetchOrganizationsQueue.take(); semaphore.acquire(); - Thread.ofVirtual().start(() -> fetchOrganizationsForClient(client)); + Thread.ofVirtual().start(() -> fetchOrganization(businessId)); } } catch (InterruptedException e) { log.warn("Interrupted while waiting for data, stopping {}", getClass().getSimpleName(), e); @@ -114,36 +107,21 @@ public void run() { } } - protected void fetchOrganizationsForClient(final ClientType client) { + protected void fetchOrganization(final String businessId) { try { - List organizationIds = OrganizationUtil.getOrganizationIdsList(client, fetchOrganizationsUrl, - fetchOrganizationsLimit, catalogService); - int numberOfOrganizations = organizationIds.size(); - log.info("Fetched {} organization GUIDs from {}", numberOfOrganizations, fetchOrganizationsUrl); - - AtomicInteger elementCount = new AtomicInteger(); - List guidsList = new ArrayList<>(); - organizationIds.forEach(id -> { - guidsList.add(id); - elementCount.getAndIncrement(); - if (elementCount.get() % maxOrganizationsPerRequest == 0) { - saveBatch(OrganizationUtil.getDataByIds(client, guidsList, fetchOrganizationsUrl, catalogService)); - guidsList.clear(); - } - if (elementCount.get() == organizationIds.size() && !guidsList.isEmpty()) { - saveBatch(OrganizationUtil.getDataByIds(client, guidsList, fetchOrganizationsUrl, catalogService)); - } - }); - log.info("Processed {} organizations", numberOfOrganizations); + log.info("Fetching organization information for member {}", businessId); + Optional organization = OrganizationUtil.getOrganization(fetchOrganizationsUrl, businessId, + catalogService); + organization.ifPresent(this::saveOrganization); } catch (Exception e) { - log.error("Error while fetching organizations for client {}", client, e); + log.error("Error while fetching organizations for member {}", businessId, e); } finally { semaphore.release(); } } - private void saveBatch(JSONArray data) { + private void saveOrganization(final JSONArray data) { for (int i = 0; i < data.length(); i++) { Organization organization = OrganizationUtil.createOrganization(data.optJSONObject(i)); JSONObject dataJson = data.optJSONObject(i); @@ -154,6 +132,7 @@ private void saveBatch(JSONArray data) { savePhoneNumbers(dataJson, savedOrganization); saveWebPages(dataJson, savedOrganization); saveAddresses(dataJson, savedOrganization); + log.info("Organization information saved for member {}", savedOrganization.getBusinessCode()); } } diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTask.java index 19ef1be7..890bed8a 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTask.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTask.java @@ -30,6 +30,7 @@ import java.util.HashSet; import java.util.List; import java.util.Queue; +import java.util.Set; import org.springframework.context.ApplicationContext; @@ -48,16 +49,16 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class ListClientsTask { +public class ListClientsTask implements Runnable { private final TaskPoolConfiguration taskPoolConfiguration; private final CatalogService catalogService; private final Queue listMethodsQueue; - private final Queue fetchCompaniesQueue; - private final Queue fetchOrganizationsQueue; + private final Queue fetchCompaniesQueue; + private final Queue fetchOrganizationsQueue; public ListClientsTask(ApplicationContext applicationContext, Queue listMethodsQueue, - Queue fetchCompaniesQueue, Queue fetchOrganizationsQueue) { + Queue fetchCompaniesQueue, Queue fetchOrganizationsQueue) { this.taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); this.catalogService = applicationContext.getBean(CatalogService.class); this.listMethodsQueue = listMethodsQueue; @@ -84,7 +85,7 @@ private void fetchClients() { log.info("Getting client list from {}", listClientsUrl); ClientList clientList = ClientListUtil.clientListFromResponse(listClientsUrl); HashMap m = populateMapWithMembers(clientList); - catalogService.saveAllMembersAndSubsystems(m.values()); + Set newMembers = catalogService.saveAllMembersAndSubsystems(m.values()); // We only fetch WSDL-s and REST services from subsystems List subsystems = clientList.getMember().stream() @@ -95,26 +96,18 @@ private void fetchClients() { log.info("All subsystems ({}) sent to ListMethodsTask", subsystems.size()); // The fetchCompaniesQueue and fetchOrganizationsQueue should only be - // initialized if the FI profile is active. The current actor implementation - // only ran these once and the specific client had no effect. This should be - // refactored once we get clarification on how this is expected to work. - if (fetchCompaniesQueue != null - && CollectorUtils.shouldFetchCompanies(taskPoolConfiguration.isFetchCompaniesRunUnlimited(), - taskPoolConfiguration.getFetchCompaniesTimeAfterHour(), - taskPoolConfiguration.getFetchCompaniesTimeBeforeHour())) { - fetchCompaniesQueue.add(clientList.getMember().getFirst()); - log.info("Notice send to the FetchCompaniesTask to do work"); + // initialized if the FI profile is active. + if (fetchCompaniesQueue != null) { + fetchCompaniesQueue.addAll(newMembers.stream().map(Member::getMemberCode).toList()); + log.info("{} new members sent to the FetchCompaniesTask", newMembers.size()); } if (fetchOrganizationsQueue != null) { - fetchOrganizationsQueue.add(clientList.getMember().getFirst()); - log.info("Notice sent to the FetchOrganizationsTask to do work"); + fetchOrganizationsQueue.addAll(newMembers.stream().map(Member::getMemberCode).toList()); + log.info("{} new members sent to the FetchOrganizationsTask", newMembers.size()); } - } catch ( - - Exception e) { + } catch (Exception e) { ErrorLog errorLog = CollectorUtils.createErrorLog(null, - "Error when fetching listClients(url: " + listClientsUrl + "): " + e.getMessage(), - "500"); + "Error when fetching listClients(url: " + listClientsUrl + "): " + e.getMessage(), "500"); catalogService.saveErrorLog(errorLog); log.error("Error when fetching listClients(url: {})", listClientsUrl, e); } @@ -126,7 +119,7 @@ private HashMap populateMapWithMembers(ClientList clientList) int clientCounter = 0; for (ClientType clientType : clientList.getMember()) { clientCounter++; - log.info("{} - {}", clientCounter, ClientTypeUtil.toString(clientType)); + log.debug("{} - {}", clientCounter, ClientTypeUtil.toString(clientType)); Member newMember = new Member(clientType.getId().getXRoadInstance(), clientType.getId() .getMemberClass(), clientType.getId().getMemberCode(), clientType.getName()); diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTask.java index dbe0eaef..bb585481 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTask.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTask.java @@ -50,7 +50,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class ListMethodsTask { +public class ListMethodsTask implements Runnable { private static final String SERVICE_TYPE_REST = "REST"; diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/UpdateExternalsTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/UpdateExternalsTask.java new file mode 100644 index 00000000..e157d29e --- /dev/null +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/UpdateExternalsTask.java @@ -0,0 +1,85 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import java.util.Queue; +import java.util.Set; + +import org.springframework.context.ApplicationContext; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.CollectorUtils; +import fi.vrk.xroad.catalog.persistence.CatalogService; +import fi.vrk.xroad.catalog.persistence.entity.ErrorLog; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class UpdateExternalsTask implements Runnable { + + private final TaskPoolConfiguration taskPoolConfiguration; + private final CatalogService catalogService; + private final Queue fetchCompaniesQueue; + private final Queue fetchOrganizationsQueue; + + public UpdateExternalsTask(ApplicationContext applicationContext, Queue fetchCompaniesQueue, + Queue fetchOrganizationsQueue) { + this.taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); + this.catalogService = applicationContext.getBean(CatalogService.class); + this.fetchCompaniesQueue = fetchCompaniesQueue; + this.fetchOrganizationsQueue = fetchOrganizationsQueue; + } + + public void run() { + if (taskPoolConfiguration.isFetchExternalRunUnlimited() + || CollectorUtils.isTimeBetweenHours(taskPoolConfiguration.getFetchExternalTimeAfterHour(), + taskPoolConfiguration.getFetchExternalTimeBeforeHour())) { + updateMemberCompanyAndOrganizations(); + } + } + + private void updateMemberCompanyAndOrganizations() { + try { + Set members = catalogService.getMembersRequiringExternalUpdate( + taskPoolConfiguration.getFetchExternalUpdateAfterDays(), + taskPoolConfiguration.getFetchExternalLimit()); + + log.info("Sending {} members requiring external update to workers, batch limit {}", members.size(), + taskPoolConfiguration.getFetchExternalLimit()); + + fetchCompaniesQueue.addAll(members); + fetchOrganizationsQueue.addAll(members); + + } catch (Exception e) { + ErrorLog errorLog = CollectorUtils.createErrorLog(null, + "Error when updating member companies and organizations: " + e.getMessage(), "500"); + catalogService.saveErrorLog(errorLog); + log.error("Error when updating member companies and organizations", e); + } + + } + +} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/ClientListUtil.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/ClientListUtil.java index c3d62cdc..de31986d 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/ClientListUtil.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/ClientListUtil.java @@ -12,15 +12,18 @@ */ package fi.vrk.xroad.catalog.collector.util; -import fi.vrk.xroad.catalog.collector.wsimport.ClientList; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import org.json.JSONArray; -import org.json.JSONObject; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; + +import java.util.List; + +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; import org.springframework.web.client.RestTemplate; +import fi.vrk.xroad.catalog.collector.wsimport.ClientList; + public final class ClientListUtil { private static final RestTemplate REST_TEMPLATE = new RestTemplate(); @@ -30,34 +33,12 @@ private ClientListUtil() { } public static ClientList clientListFromResponse(String url) { - JSONArray members = new JSONArray(); - - ResponseEntity response = REST_TEMPLATE.getForEntity(url, String.class); - JSONObject json = new JSONObject(response.getBody()); - members = json.getJSONArray("member"); - - ClientList clientList = new ClientList(); - for (int i = 0; i < members.length(); i++) { - ClientType clientType = new ClientType(); - JSONObject id = members.getJSONObject(i).getJSONObject("id"); - XRoadClientIdentifierType xRoadClientIdentifierType = new XRoadClientIdentifierType(); - xRoadClientIdentifierType.setXRoadInstance(id.optString("xroad_instance")); - xRoadClientIdentifierType.setMemberClass(id.optString("member_class")); - xRoadClientIdentifierType.setMemberCode(id.optString("member_code")); - xRoadClientIdentifierType.setSubsystemCode(id.optString("subsystem_code")); - xRoadClientIdentifierType.setGroupCode(id.optString("group_code")); - xRoadClientIdentifierType.setServiceCode(id.optString("service_code")); - xRoadClientIdentifierType - .setServiceVersion(id.has("service_version") ? id.optString("service_version") : null); - xRoadClientIdentifierType.setSecurityCategoryCode(id.optString("security_category_code")); - xRoadClientIdentifierType.setServerCode(id.optString("server_code")); - xRoadClientIdentifierType.setObjectType(XRoadObjectType.fromValue(id.optString("object_type"))); - clientType.setId(xRoadClientIdentifierType); - clientType.setName((String) members.getJSONObject(i).get("name")); - clientList.getMember().add(clientType); - } - - return clientList; + HttpHeaders headers = new HttpHeaders(); + headers.setAccept(List.of(MediaType.TEXT_XML)); + HttpEntity requestEntity = new HttpEntity<>(headers); + ResponseEntity response = REST_TEMPLATE.exchange(url, HttpMethod.GET, requestEntity, + ClientList.class); + return response.getBody(); } } diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/OrganizationUtil.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/OrganizationUtil.java index 67763207..bf87d744 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/OrganizationUtil.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/OrganizationUtil.java @@ -12,7 +12,6 @@ */ package fi.vrk.xroad.catalog.collector.util; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; import fi.vrk.xroad.catalog.persistence.CatalogService; import fi.vrk.xroad.catalog.persistence.entity.Address; import fi.vrk.xroad.catalog.persistence.entity.BusinessAddress; @@ -61,10 +60,14 @@ import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + import javax.net.ssl.SSLContext; import java.security.KeyManagementException; import java.security.KeyStoreException; @@ -75,13 +78,13 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Optional; @Slf4j public final class OrganizationUtil { - private static final String TOTAL_RESULTS = "true"; - private static final String RESULTS_FROM = "0"; - private static final String REGISTRATION_FROM = "1970-01-01"; + private static final RestTemplate REST_TEMPLATE = createTemplate(); + private static final String WITH_BUSINESS_CODE = "with businessCode "; private static final String DESCRIPTION = "description"; private static final String LANGUAGE = "language"; @@ -96,156 +99,76 @@ private OrganizationUtil() { } - public static JSONObject getCompanies(ClientType clientType, - Integer fetchCompaniesLimit, - String url, - CatalogService catalogService) - throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - final String fetchCompaniesUrl = new StringBuilder() - .append(url) - .append("?totalResults=").append(TOTAL_RESULTS) - .append("&maxResults=").append(String.valueOf(fetchCompaniesLimit)) - .append("&resultsFrom=").append(RESULTS_FROM) - .append("&companyRegistrationFrom=").append(REGISTRATION_FROM) - .toString(); + public static Optional getCompany(String url, String businessId, CatalogService catalogService) + throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException { + final String fetchCompaniesUrl = UriComponentsBuilder.fromHttpUrl(url).pathSegment(businessId).encode() + .build().toString(); JSONObject jsonObject = new JSONObject(); try { - String ret = getResponseBody(fetchCompaniesUrl); + String ret = getResponseBody(fetchCompaniesUrl, String.class); jsonObject = new JSONObject(ret); - return jsonObject; - } catch (KeyStoreException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "KeyStoreException occurred when fetching list of companies from url " + url, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("KeyStoreException occurred when fetching companies from url {}", url); - throw e; - } catch (NoSuchAlgorithmException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "NoSuchAlgorithmException occurred when fetching companies from url " + url, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("NoSuchAlgorithmException occurred when fetching companies from url", url); - throw e; - } catch (KeyManagementException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "KeyManagementException occurred when fetching companies from url " + url, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("KeyManagementException occurred when fetching companies from url {}", url); - throw e; - } catch (Exception e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "Exception occurred when fetching companies from url " + url, + return Optional.of(jsonObject); + } catch (HttpClientErrorException e) { + // This is not an error, since not all institutions exist in this registry + if (e.getStatusCode().isSameCodeAs(HttpStatus.NOT_FOUND)) { + log.warn("Company with businessId {} not found at {}", businessId, fetchCompaniesUrl); + return Optional.empty(); + } + ErrorLog errorLog = CollectorUtils.createErrorLog(null, + "HttpClientErrorException occurred when fetching organization from url " + url + + WITH_BUSINESS_CODE + + businessId, "500"); catalogService.saveErrorLog(errorLog); - log.error("Exception occurred when fetching companies from url {}", url); + log.error("HttpClientErrorException occurred when fetching organization from url {} with businessCode {}", + url, + businessId); throw e; - } - } - - public static JSONObject getCompany(ClientType clientType, String url, String businessCode, - CatalogService catalogService) { - final String fetchCompaniesUrl = new StringBuilder().append(url) - .append("/").append(businessCode).toString(); - JSONObject jsonObject = new JSONObject(); - try { - String ret = getResponseBody(fetchCompaniesUrl); - jsonObject = new JSONObject(ret); - return jsonObject; - } catch (KeyStoreException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "KeyStoreException occurred when fetching companies from url " + url - + WITH_BUSINESS_CODE + businessCode, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("KeyStoreException occurred when fetching companies from url {} with businessCode {}", - url, businessCode); - } catch (NoSuchAlgorithmException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "NoSuchAlgorithmException occurred when fetching companies from url " + url - + WITH_BUSINESS_CODE + businessCode, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("NoSuchAlgorithmException occurred when fetching companies from url {} with businessCode {}", - url, businessCode); - } catch (KeyManagementException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "KeyManagementException occurred when fetching companies from url " + url - + WITH_BUSINESS_CODE + businessCode, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("KeyManagementException occurred when fetching companies from url {} with businessCode {}", - url, businessCode); } catch (Exception e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(null, "Exception occurred when fetching companies from url " + url - + WITH_BUSINESS_CODE + businessCode, + + WITH_BUSINESS_CODE + businessId, "500"); catalogService.saveErrorLog(errorLog); log.error("Exception occurred when fetching companies from url {} with businessCode {}", url, - businessCode); + businessId); + throw e; } - return jsonObject; } - public static List getOrganizationIdsList(ClientType clientType, String url, - Integer fetchOrganizationsLimit, CatalogService catalogService) - throws KeyStoreException, NoSuchAlgorithmException, KeyManagementException { + public static Optional getOrganization(String url, String businessId, CatalogService catalogService) + throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException { + final String fetchOrganizationUrl = UriComponentsBuilder.fromHttpUrl(url) + .pathSegment("businesscode", businessId).encode().build().toString(); try { - List idsList = new ArrayList<>(); - String response = getResponseBody(url); - JSONObject json = new JSONObject(response); - JSONArray itemList = json.optJSONArray("itemList"); - int totalFetchAmount = itemList.length() > fetchOrganizationsLimit ? fetchOrganizationsLimit - : itemList.length(); - for (int i = 0; i < totalFetchAmount; i++) { - String id = itemList.optJSONObject(i).optString("id"); - idsList.add(id); + String ret = getResponseBody(fetchOrganizationUrl, String.class); + JSONArray jsonObject = new JSONArray(ret); + return Optional.of(jsonObject); + } catch (HttpClientErrorException e) { + // This is not an error, since not all institutions exist in this registry + if (e.getStatusCode().isSameCodeAs(HttpStatus.NOT_FOUND)) { + log.warn("Organization with businessId {} not found at {}", businessId, + fetchOrganizationUrl); + return Optional.empty(); } - return idsList; - } catch (KeyStoreException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "KeyStoreException occurred when fetching organization ids with from url " - + url, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("KeyStoreException occurred when fetching organization ids with from url {}", url); - throw e; - } catch (NoSuchAlgorithmException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "NoSuchAlgorithmException occurred when fetching organization ids with from url " - + url, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("NoSuchAlgorithmException occurred when fetching organization ids with from url {}", - url); - throw e; - } catch (KeyManagementException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "KeyManagementException occurred when fetching organization ids with from url " - + url, + ErrorLog errorLog = CollectorUtils.createErrorLog(null, + "HttpClientErrorException occurred when fetching organization from url " + url + + WITH_BUSINESS_CODE + + businessId, "500"); catalogService.saveErrorLog(errorLog); - log.error("KeyManagementException occurred when fetching organizations with from url {}", url); + log.error("HttpClientErrorException occurred when fetching organization from url {} with businessCode {}", + url, + businessId); throw e; } catch (Exception e) { - log.error("Exception occurred when fetching organization ids: " + e.getMessage()); - ErrorLog errorLog = ErrorLog.builder() - .created(LocalDateTime.now()) - .message("Exception occurred when fetching organization ids: " + e.getMessage()) - .code("500") - .xRoadInstance(clientType.getId().getXRoadInstance()) - .memberClass(clientType.getId().getMemberClass()) - .memberCode(clientType.getId().getMemberCode()) - .groupCode(clientType.getId().getGroupCode()) - .securityCategoryCode(clientType.getId().getSecurityCategoryCode()) - .serverCode(clientType.getId().getServerCode()) - .serviceCode(clientType.getId().getServiceCode()) - .serviceVersion(clientType.getId().getServiceVersion()) - .subsystemCode(clientType.getId().getSubsystemCode()) - .build(); + ErrorLog errorLog = CollectorUtils.createErrorLog(null, + "Exception occurred when fetching organization from url " + url + + WITH_BUSINESS_CODE + businessId, + "500"); catalogService.saveErrorLog(errorLog); + log.error("Exception occurred when fetching organization from url {} with businessCode {}", url, + businessId); throw e; } } @@ -664,106 +587,47 @@ private static LocalDateTime parseDateFromString(String dateValue) { return null; } - public static JSONArray getDataByIds(ClientType clientType, List guids, String url, - CatalogService catalogService) { - String requestGuids = ""; - for (int i = 0; i < guids.size(); i++) { - requestGuids += guids.get(i); - if (i < (guids.size() - 1)) { - requestGuids += ","; - } - } - - final String listOrganizationsUrl = new StringBuilder().append(url) - .append("/list?guids=").append(requestGuids).toString(); - - log.debug("Fetching organizations with guids: " + listOrganizationsUrl); - - JSONArray itemList = new JSONArray(); - try { - String ret = getResponseBody(listOrganizationsUrl); - JSONObject json = new JSONObject("{\"items\":" + ret + "}"); - itemList = json.optJSONArray("items"); - return itemList; - } catch (KeyStoreException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "KeyStoreException occurred when fetching organizations with from url " + url, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("KeyStoreException occurred when fetching organizations with from url {}", url); - } catch (NoSuchAlgorithmException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "NoSuchAlgorithmException occurred when fetching organizations with from url " - + url, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("NoSuchAlgorithmException occurred when fetching organizations with from url {}", - url); - } catch (KeyManagementException e) { - ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, - "KeyManagementException occurred when fetching organizations with from url " - + url, - "500"); - catalogService.saveErrorLog(errorLog); - log.error("KeyManagementException occurred when fetching organizations with from url {}", url); - } catch (Exception e) { - log.error("Exception occurred when fetching organization data: " + e.getMessage()); - ErrorLog errorLog = ErrorLog.builder() - .created(LocalDateTime.now()) - .message("Exception occurred when fetching organization data: " - + e.getMessage()) - .code("500") - .xRoadInstance(clientType.getId().getXRoadInstance()) - .memberClass(clientType.getId().getMemberClass()) - .memberCode(clientType.getId().getMemberCode()) - .groupCode(clientType.getId().getGroupCode()) - .serverCode(clientType.getId().getServerCode()) - .serviceCode(clientType.getId().getServiceCode()) - .serviceVersion(clientType.getId().getServiceVersion()) - .subsystemCode(clientType.getId().getSubsystemCode()) - .build(); - catalogService.saveErrorLog(errorLog); + private static T getResponseBody(String url, Class returnType) { + if (REST_TEMPLATE == null) { + throw new CatalogCollectorRuntimeException( + "OrganizationUtil RestTemplate not initialized, please check startup logs"); } - return itemList; - } - - public static String getResponseBody(String url) - throws KeyStoreException, NoSuchAlgorithmException, - KeyManagementException { HttpHeaders headers = new HttpHeaders(); List mediaTypes = new ArrayList<>(); mediaTypes.add(MediaType.APPLICATION_JSON); headers.setAccept(mediaTypes); final HttpEntity entity = new HttpEntity<>(headers); - RestTemplate restTemplate = createTemplate(); - ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, entity, String.class); - + ResponseEntity response = REST_TEMPLATE.exchange(url, HttpMethod.GET, entity, returnType); return response.getBody(); } - private static RestTemplate createTemplate() - throws KeyStoreException, NoSuchAlgorithmException, KeyManagementException { - TrustStrategy acceptingTrustStrategy = new TrustStrategy() { - @Override - public boolean isTrusted(X509Certificate[] x509Certificates, String s) - throws CertificateException { - return true; - } - }; - SSLContext sslContext = SSLContexts.custom() - .loadTrustMaterial(null, acceptingTrustStrategy) - .build(); - SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext, - new NoopHostnameVerifier()); - PoolingHttpClientConnectionManager cm = PoolingHttpClientConnectionManagerBuilder - .create() - .setSSLSocketFactory(csf) - .build(); - CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build(); - HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(); - requestFactory.setHttpClient(httpClient); - RestTemplate restTemplate = new RestTemplate(requestFactory); - - return restTemplate; + private static RestTemplate createTemplate() { + try { + TrustStrategy acceptingTrustStrategy = new TrustStrategy() { + @Override + public boolean isTrusted(X509Certificate[] x509Certificates, String s) + throws CertificateException { + return true; + } + }; + SSLContext sslContext = SSLContexts.custom() + .loadTrustMaterial(null, acceptingTrustStrategy) + .build(); + SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext, + new NoopHostnameVerifier()); + PoolingHttpClientConnectionManager cm = PoolingHttpClientConnectionManagerBuilder + .create() + .setSSLSocketFactory(csf) + .build(); + CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build(); + HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(); + requestFactory.setHttpClient(httpClient); + RestTemplate restTemplate = new RestTemplate(requestFactory); + + return restTemplate; + } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + log.error("Error creating REST client for Company and Organization services", e); + return null; + } } } diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTaskTest.java index 14174c27..e39ff556 100644 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTaskTest.java +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTaskTest.java @@ -34,6 +34,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Optional; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; @@ -55,9 +56,6 @@ import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; import fi.vrk.xroad.catalog.collector.util.OrganizationUtil; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; import fi.vrk.xroad.catalog.persistence.CatalogService; import fi.vrk.xroad.catalog.persistence.CompanyService; @@ -74,9 +72,6 @@ public class FetchCompaniesTaskTest { @Autowired private ApplicationContext applicationContext; - @Value("classpath:mock/companies/getCompanies.json") - private Resource companiesJSON; - @Value("classpath:mock/companies/company.json") private Resource companyJSON; @@ -89,13 +84,12 @@ public void testBasicNoDeadlock() throws InterruptedException { * sure that the task can also be stopped when the program exits. The actual * fetch logic is mocked and tested below. */ - BlockingQueue queue = new LinkedBlockingQueue<>(); + BlockingQueue queue = new LinkedBlockingQueue<>(); FetchCompaniesTask fetchCompaniesTask = new FetchCompaniesTask(applicationContext, queue); Semaphore semaphore = new Semaphore(1); ReflectionTestUtils.setField(fetchCompaniesTask, "semaphore", semaphore); - ClientType clientType = new ClientType(); Thread fetchCompaniesRunner = Thread.ofVirtual().start(fetchCompaniesTask::run); - queue.add(clientType); + queue.add(""); Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> queue.isEmpty()); @@ -106,36 +100,32 @@ public void testBasicNoDeadlock() throws InterruptedException { } @Test - public void testFetchCompaniesForClient() throws JSONException, IOException { + public void testFetchCompanyForClient() throws JSONException, IOException { try (MockedStatic mock = Mockito.mockStatic(OrganizationUtil.class)) { FetchCompaniesTask fetchCompaniesTask = new FetchCompaniesTask(applicationContext, null); - final JSONObject getCompaniesResponse = new JSONObject( - companiesJSON.getContentAsString(StandardCharsets.UTF_8)); - mock.when(() -> OrganizationUtil.getCompanies(any(), any(), any(), any())) - .thenReturn(getCompaniesResponse); - - final JSONObject getCompanyResponse = new JSONObject( - companyJSON.getContentAsString(StandardCharsets.UTF_8)); - mock.when(() -> OrganizationUtil.getCompany(any(), any(), any(), any())) - .thenReturn(getCompanyResponse); - ClientType clientType = new ClientType(); - XRoadClientIdentifierType value = new XRoadClientIdentifierType(); - value.setXRoadInstance("INSTANCE"); - value.setMemberClass("COM"); - value.setMemberCode("1234567-9"); - value.setSubsystemCode("SUBSYSTEM"); - value.setServiceCode("aService"); - value.setServiceVersion("v1"); - value.setObjectType(XRoadObjectType.SERVICE); - clientType.setId(value); - - fetchCompaniesTask.fetchCompaniesForCLient(clientType); - - mock.verify(() -> OrganizationUtil.getCompanies(any(), any(), any(), any()), times(1)); - mock.verify(() -> OrganizationUtil.getCompany(any(), any(), any(), any()), times(4)); - verify(companyService, times(4)).saveCompany(any()); + final Optional getCompanyResponse = Optional.ofNullable(new JSONObject( + companyJSON.getContentAsString(StandardCharsets.UTF_8))); + mock.when(() -> OrganizationUtil.getCompany(any(), any(), any())).thenReturn(getCompanyResponse); + + fetchCompaniesTask.fetchCompanyData("1234567-9"); + + mock.verify(() -> OrganizationUtil.getCompany(any(), any(), any()), times(1)); + verify(companyService, times(1)).saveCompany(any()); } } + @Test + public void testFetchCompanyForClientNotFound() throws JSONException, IOException { + try (MockedStatic mock = Mockito.mockStatic(OrganizationUtil.class)) { + FetchCompaniesTask fetchCompaniesTask = new FetchCompaniesTask(applicationContext, null); + + mock.when(() -> OrganizationUtil.getCompany(any(), any(), any())).thenReturn(Optional.empty()); + + fetchCompaniesTask.fetchCompanyData("1234567-9"); + + mock.verify(() -> OrganizationUtil.getCompany(any(), any(), any()), times(1)); + verify(companyService, times(0)).saveCompany(any()); + } + } } diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationTaskTest.java index 6aa463d3..2484880f 100644 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationTaskTest.java +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationTaskTest.java @@ -34,7 +34,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.List; +import java.util.Optional; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; @@ -51,22 +51,14 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.ApplicationContext; import org.springframework.core.io.Resource; -import org.springframework.test.context.TestPropertySource; import org.springframework.test.util.ReflectionTestUtils; import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; import fi.vrk.xroad.catalog.collector.util.OrganizationUtil; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; import fi.vrk.xroad.catalog.persistence.CatalogService; import fi.vrk.xroad.catalog.persistence.OrganizationService; @SpringBootTest(classes = TaskPoolConfiguration.class) -@TestPropertySource(properties = { - "xroad-catalog.fetch-organizations-limit=3", - "xroad-catalog.max-organizations-per-request=3" -}) public class FetchOrganizationTaskTest { @MockBean @@ -90,13 +82,12 @@ public void testBasicNoDeadlock() throws InterruptedException { * sure that the task can also be stopped when the program exits. The actual * fetch logic is mocked and tested below. */ - BlockingQueue queue = new LinkedBlockingQueue<>(); + BlockingQueue queue = new LinkedBlockingQueue<>(); FetchOrganizationsTask fetchOrganizationsTask = new FetchOrganizationsTask(applicationContext, queue); Semaphore semaphore = new Semaphore(1); ReflectionTestUtils.setField(fetchOrganizationsTask, "semaphore", semaphore); - ClientType clientType = new ClientType(); Thread fetchOrganizationsRunner = Thread.ofVirtual().start(fetchOrganizationsTask::run); - queue.add(clientType); + queue.add(""); Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> queue.isEmpty()); semaphore.acquire(); fetchOrganizationsRunner.interrupt(); @@ -109,34 +100,30 @@ public void testFetchOrganizationsForClient() throws JSONException, IOException FetchOrganizationsTask fetchOrganizationsTask = new FetchOrganizationsTask(applicationContext, null); - mock.when(() -> OrganizationUtil.getOrganizationIdsList(any(), any(), any(), any())) - .thenReturn(List.of( - "112ea34r3-1k23-412r-9142-1442asd13131", - "112ea34r3-1k23-412r-9142-1442asd13132", - "112ea34r3-1k23-412r-9142-1442asd13133")); - - JSONArray organizationsByIdResponse = new JSONArray( - organizationsByIdJSON.getContentAsString(StandardCharsets.UTF_8)); - mock.when(() -> OrganizationUtil.getDataByIds(any(), any(), any(), any())) + Optional organizationsByIdResponse = Optional.ofNullable(new JSONArray( + organizationsByIdJSON.getContentAsString(StandardCharsets.UTF_8))); + mock.when(() -> OrganizationUtil.getOrganization(any(), any(), any())) .thenReturn(organizationsByIdResponse); - ClientType clientType = new ClientType(); - XRoadClientIdentifierType value = new XRoadClientIdentifierType(); - value.setXRoadInstance("INSTANCE"); - value.setMemberClass("CLASS"); - value.setMemberCode("CODE"); - value.setSubsystemCode("SUBSYSTEM"); - value.setServiceCode("aService"); - value.setServiceVersion("v1"); - value.setObjectType(XRoadObjectType.SERVICE); - clientType.setId(value); - - fetchOrganizationsTask.fetchOrganizationsForClient(clientType); - - mock.verify(() -> OrganizationUtil.getOrganizationIdsList(any(), any(), any(), any()), - times(1)); - mock.verify(() -> OrganizationUtil.getDataByIds(any(), any(), any(), any()), times(1)); - verify(organizationService, times(3)).saveOrganization(any()); + fetchOrganizationsTask.fetchOrganization("1234"); + + mock.verify(() -> OrganizationUtil.getOrganization(any(), any(), any()), times(1)); + verify(organizationService, times(1)).saveOrganization(any()); + } + } + + @Test + public void testFetchOrganizationsForClientNotFound() throws JSONException, IOException { + try (MockedStatic mock = Mockito.mockStatic(OrganizationUtil.class)) { + FetchOrganizationsTask fetchOrganizationsTask = new FetchOrganizationsTask(applicationContext, + null); + + mock.when(() -> OrganizationUtil.getOrganization(any(), any(), any())).thenReturn(Optional.empty()); + + fetchOrganizationsTask.fetchOrganization("1234"); + + mock.verify(() -> OrganizationUtil.getOrganization(any(), any(), any()), times(1)); + verify(organizationService, times(0)).saveOrganization(any()); } } } diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTaskTest.java index 7988a120..d535747e 100644 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTaskTest.java +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTaskTest.java @@ -33,6 +33,7 @@ import static org.mockito.Mockito.verify; import java.util.Queue; +import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import org.junit.jupiter.api.Test; @@ -51,6 +52,7 @@ import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; import fi.vrk.xroad.catalog.persistence.CatalogService; +import fi.vrk.xroad.catalog.persistence.entity.Member; @SpringBootTest(classes = TaskPoolConfiguration.class) public class ListClientsTaskTest { @@ -68,7 +70,6 @@ public void testOnReceiveWhenFetchUnlimited() throws Exception { TaskPoolConfiguration conf = applicationContext.getBean(TaskPoolConfiguration.class); ReflectionTestUtils.setField(conf, "fetchRunUnlimited", true); - ReflectionTestUtils.setField(conf, "fetchCompaniesRunUnlimited", true); ClientList clientList = new ClientList(); clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member1", null)); @@ -83,8 +84,14 @@ public void testOnReceiveWhenFetchUnlimited() throws Exception { .thenReturn(clientList); final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + + final Member member1 = new Member(); + member1.setMemberCode("member1"); + final Member member2 = new Member(); + member2.setMemberCode("member2"); + Mockito.when(catalogService.saveAllMembersAndSubsystems(any())).thenReturn(Set.of(member1, member2)); ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, fetchCompaniesQueue, fetchOrganisationsQueue); @@ -93,8 +100,8 @@ public void testOnReceiveWhenFetchUnlimited() throws Exception { verify(catalogService, times(1)).saveAllMembersAndSubsystems(any()); assertEquals(5, listMethodsQueue.size()); - assertEquals(1, fetchCompaniesQueue.size()); - assertEquals(1, fetchOrganisationsQueue.size()); + assertEquals(2, fetchCompaniesQueue.size()); + assertEquals(2, fetchOrganisationsQueue.size()); } } @@ -119,8 +126,14 @@ public void testWhenFetchNotUnlimitedAndTimeOutsideOfConfiguration() throws Exce mocked.when(() -> ClientListUtil.clientListFromResponse(any())).thenReturn(clientList); final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + + final Member member1 = new Member(); + member1.setMemberCode("member1"); + final Member member2 = new Member(); + member2.setMemberCode("member2"); + Mockito.when(catalogService.saveAllMembersAndSubsystems(any())).thenReturn(Set.of(member1, member2)); ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, fetchCompaniesQueue, fetchOrganisationsQueue); @@ -143,10 +156,6 @@ public void testOnReceiveWhenFetchNotUnlimitedButTimeIsInBetween() throws Except ReflectionTestUtils.setField(conf, "fetchTimeAfterHour", 0); ReflectionTestUtils.setField(conf, "fetchTimeBeforeHour", 23); - ReflectionTestUtils.setField(conf, "fetchCompaniesRunUnlimited", false); - ReflectionTestUtils.setField(conf, "fetchCompaniesTimeAfterHour", 0); - ReflectionTestUtils.setField(conf, "fetchCompaniesTimeBeforeHour", 23); - ClientList clientList = new ClientList(); clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member1", null)); clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub1")); @@ -159,8 +168,14 @@ public void testOnReceiveWhenFetchNotUnlimitedButTimeIsInBetween() throws Except mocked.when(() -> ClientListUtil.clientListFromResponse(any())).thenReturn(clientList); final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + + final Member member1 = new Member(); + member1.setMemberCode("member1"); + final Member member2 = new Member(); + member2.setMemberCode("member2"); + Mockito.when(catalogService.saveAllMembersAndSubsystems(any())).thenReturn(Set.of(member1, member2)); ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, fetchCompaniesQueue, fetchOrganisationsQueue); @@ -169,8 +184,8 @@ public void testOnReceiveWhenFetchNotUnlimitedButTimeIsInBetween() throws Except verify(catalogService, times(1)).saveAllMembersAndSubsystems(any()); assertEquals(5, listMethodsQueue.size()); - assertEquals(1, fetchCompaniesQueue.size()); - assertEquals(1, fetchOrganisationsQueue.size()); + assertEquals(2, fetchCompaniesQueue.size()); + assertEquals(2, fetchOrganisationsQueue.size()); } } @@ -185,8 +200,10 @@ public void testOnReceiveWithEmptyMemberList() throws Exception { mocked.when(() -> ClientListUtil.clientListFromResponse(any())).thenReturn(clientList); final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + + Mockito.when(catalogService.saveAllMembersAndSubsystems(any())).thenReturn(Set.of()); ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, fetchCompaniesQueue, fetchOrganisationsQueue); @@ -206,8 +223,8 @@ public void testSaveErrorLog() { ReflectionTestUtils.setField(conf, "fetchRunUnlimited", true); final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); - final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, fetchCompaniesQueue, fetchOrganisationsQueue); diff --git a/xroad-catalog-collector/src/test/resources/mock/companies/getCompanies.json b/xroad-catalog-collector/src/test/resources/mock/companies/getCompanies.json deleted file mode 100644 index d222774f..00000000 --- a/xroad-catalog-collector/src/test/resources/mock/companies/getCompanies.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "type":"fi.prh.opendata.bis", - "version":"1", - "totalResults":4, - "resultsFrom":0, - "previousResultsUri":null, - "nextResultsUri":"http://localhost/v1?totalResults=true&maxResults=100&resultsFrom=100&companyRegistrationFrom=1970-01-01", - "exceptionNoticeUri":null, - "results":[ - { - "businessId":"1234567-1", - "name":"Test Company 1", - "registrationDate":"2020-01-01", - "companyForm":"OY", - "detailsUri":"http://localhost/v1/1234567-1" - }, - { - "businessId":"1234567-2", - "name":"Test Company 2", - "registrationDate":"2022-02-09", - "companyForm":"OY", - "detailsUri":"http://localhost/v1/1234567-2" - }, - { - "businessId":"1234567-3", - "name":"Test Company 3", - "registrationDate":"2012-06-08", - "companyForm":"OY", - "detailsUri":"http://localhost/v1/1234567-3" - }, - { - "businessId":"1234567-4", - "name":"Test Company 4", - "registrationDate":"2019-05-09", - "companyForm":"OY", - "detailsUri":"http://localhost/v1/1234567-4" - } - ] -} diff --git a/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/CatalogService.java b/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/CatalogService.java index 6b4d8a90..f94146f8 100644 --- a/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/CatalogService.java +++ b/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/CatalogService.java @@ -30,6 +30,7 @@ import java.time.LocalDateTime; import java.util.Collection; import java.util.List; +import java.util.Set; /** * CRUD methods for catalog objects. no business logic (e.g. hash calculation), @@ -233,8 +234,9 @@ List getDistinctServiceStatistics(LocalDateTime start * member.subsystems collection populated, and each subsystem * should * have subsystem.member populated as well. + * @return Set of Member entities representing new members saved to the database */ - void saveAllMembersAndSubsystems(Collection members); + Set saveAllMembersAndSubsystems(Collection members); /** * Stores services for given subsystem. Does not modify the associated Subsystem @@ -339,4 +341,15 @@ List getDistinctServiceStatistics(LocalDateTime start */ void deleteOldErrorLogEntries(Integer daysBefore); + /** + * Returns a batch of Member codes for members that haven't had their Company or + * Organization data updated for + * the specified number of days. + * + * @param daysSinceLastUpdate number of days since last update + * @param batchSize number of items to return + * @return List of Member codes + */ + Set getMembersRequiringExternalUpdate(int daysSinceLastUpdate, int batchSize); + } diff --git a/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/CatalogServiceImpl.java b/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/CatalogServiceImpl.java index 4378251f..ab4fdc23 100644 --- a/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/CatalogServiceImpl.java +++ b/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/CatalogServiceImpl.java @@ -40,7 +40,6 @@ import fi.vrk.xroad.catalog.persistence.repository.ServiceRepository; import fi.vrk.xroad.catalog.persistence.repository.SubsystemRepository; import fi.vrk.xroad.catalog.persistence.repository.WsdlRepository; -import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; @@ -51,6 +50,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -61,7 +61,6 @@ /** * Implementation for catalogservice CRUD */ -@Slf4j @Component("catalogService") @Transactional public class CatalogServiceImpl implements CatalogService { @@ -287,18 +286,15 @@ public List getDistinctServiceStatistics(LocalDateTim List services = serviceRepository.findAllActive(); LocalDateTime dateInPast = startDateTime; while (isDateBetweenDates(dateInPast, startDateTime, endDateTime)) { - // TODO: Why are we using AtomicLong here? - AtomicLong totalDistinctServices = new AtomicLong(); + long totalDistinctServices = 0; List servicesBetweenDates = services.stream() .filter(p -> p.getStatusInfo().getCreated().isBefore(endDateTime)) .toList(); if (!servicesBetweenDates.isEmpty()) { - totalDistinctServices - .set(servicesBetweenDates.stream().map(Service::getServiceCode).distinct().count()); + totalDistinctServices = servicesBetweenDates.stream().map(Service::getServiceCode).distinct().count(); - DistinctServiceStatistics serviceStatistics = DistinctServiceStatistics.builder() - .created(dateInPast) - .numberOfDistinctServices(totalDistinctServices.longValue()).build(); + DistinctServiceStatistics serviceStatistics = DistinctServiceStatistics.builder().created(dateInPast) + .numberOfDistinctServices(totalDistinctServices).build(); serviceStatisticsList.add(serviceStatistics); } @@ -368,17 +364,19 @@ public Iterable getErrorLog(LocalDateTime startDateTime, LocalDateTime } @Override - public void saveAllMembersAndSubsystems(Collection members) { + public Set saveAllMembersAndSubsystems(Collection members) { LocalDateTime now = LocalDateTime.now(); // process members Map unprocessedOldMembers = new HashMap<>(); StreamSupport.stream(memberRepository.findAll().spliterator(), false) .forEach(member -> unprocessedOldMembers.put(member.createKey(), member)); + Set newMembers = new HashSet<>(); for (Member member : members) { Member oldMember = unprocessedOldMembers.get(member.createKey()); if (oldMember == null) { // brand new item + newMembers.add(member); member.getStatusInfo().setTimestampsForNew(now); for (Subsystem subsystem : member.getAllSubsystems()) { subsystem.getStatusInfo().setTimestampsForNew(now); @@ -395,6 +393,7 @@ public void saveAllMembersAndSubsystems(Collection members) { // now unprocessedOldMembers should all be removed (either already removed, or // will be now) removeUnprocessedOldMembers(now, unprocessedOldMembers); + return newMembers; } @Override @@ -605,6 +604,11 @@ public LastCollectionData getLastCollectionData() { .wsdlsLastFetched(wsdlRepository.findLatestFetched()).build(); } + @Override + public Set getMembersRequiringExternalUpdate(int daysSinceLastUpdate, int batchSize) { + return memberRepository.findMembersRequiringExternalUpdate(daysSinceLastUpdate, batchSize); + } + private void handleOldMember(LocalDateTime now, Member member, Member oldMember) { oldMember.updateWithDataFrom(member, now); // process subsystems for the old member diff --git a/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/repository/MemberRepository.java b/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/repository/MemberRepository.java index 9d25d30c..0042482f 100644 --- a/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/repository/MemberRepository.java +++ b/xroad-catalog-persistence/src/main/java/fi/vrk/xroad/catalog/persistence/repository/MemberRepository.java @@ -27,12 +27,10 @@ public interface MemberRepository extends CrudRepository { - @EntityGraph(value = "member.full-tree.graph", - type = EntityGraph.EntityGraphType.FETCH) + @EntityGraph(value = "member.full-tree.graph", type = EntityGraph.EntityGraphType.FETCH) Set findAll(); - @EntityGraph(value = "member.full-tree.graph", - type = EntityGraph.EntityGraphType.FETCH) + @EntityGraph(value = "member.full-tree.graph", type = EntityGraph.EntityGraphType.FETCH) @Query("SELECT m FROM Member m WHERE m.statusInfo.removed IS NULL") Set findAllActive(); @@ -47,16 +45,18 @@ public interface MemberRepository extends CrudRepository { // uses named query Member.findAllChangedBetween Set findAllChangedBetween(@Param("startDate") LocalDateTime startDate, - @Param("endDate") LocalDateTime endDate); + @Param("endDate") LocalDateTime endDate); // uses named query Member.findActiveChangedBetween Set findActiveChangedBetween(@Param("startDate") LocalDateTime startDate, - @Param("endDate") LocalDateTime endDate); + @Param("endDate") LocalDateTime endDate); + /** * Returns only active items (non-deleted) + * * @param xRoadInstance X-Road instance parameter, for example FI - * @param memberClass X-Road member class, for example GOF - * @param memberCode X-Road member class, for example Company code + * @param memberClass X-Road member class, for example GOF + * @param memberCode X-Road member class, for example Company code * @return Member found */ @Query("SELECT m FROM Member m WHERE m.xRoadInstance = :xRoadInstance " @@ -64,8 +64,8 @@ Set findActiveChangedBetween(@Param("startDate") LocalDateTime startDate + "AND m.memberCode = :memberCode " + "AND m.statusInfo.removed IS NULL") Member findActiveByNaturalKey(@Param("xRoadInstance") String xRoadInstance, - @Param("memberClass") String memberClass, - @Param("memberCode") String memberCode); + @Param("memberClass") String memberClass, + @Param("memberCode") String memberCode); @Query(value = "SELECT 1", nativeQuery = true) Integer checkConnection(); @@ -73,4 +73,14 @@ Member findActiveByNaturalKey(@Param("xRoadInstance") String xRoadInstance, @Query(value = "SELECT MAX(fetched) FROM member", nativeQuery = true) LocalDateTime findLatestFetched(); + @Query(value = "SELECT mem.member_code" + + " FROM member mem" + + " LEFT JOIN company com ON mem.member_code = com.business_id" + + " LEFT JOIN organization org ON mem.member_code = org.business_code" + + " WHERE DATE_PART('day', (now() - LEAST(mem.fetched, com.fetched, org.fetched))) >= :unchangedForDays" + + " ORDER BY LEAST(mem.fetched, com.fetched, org.fetched) ASC" + + " LIMIT :batchLimit", nativeQuery = true) + Set findMembersRequiringExternalUpdate(@Param("unchangedForDays") Integer unchangedForDays, + @Param("batchLimit") Integer batchLimit); + }