From 9bb877daa758f9f98dbfef1219f339600cb36f36 Mon Sep 17 00:00:00 2001 From: Minkyu Park Date: Mon, 19 Sep 2022 10:47:25 -0700 Subject: [PATCH 01/25] Notify run transition event to the listener (#2125) Signed-off-by: Minkyu Park Signed-off-by: Minkyu Park --- .../marquez/service/OpenLineageService.java | 15 +++++++++++++++ .../OpenLineageServiceIntegrationTest.java | 17 +++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/api/src/main/java/marquez/service/OpenLineageService.java b/api/src/main/java/marquez/service/OpenLineageService.java index b8ac4ec64f..9c7deb4e24 100644 --- a/api/src/main/java/marquez/service/OpenLineageService.java +++ b/api/src/main/java/marquez/service/OpenLineageService.java @@ -30,6 +30,7 @@ import marquez.common.models.JobVersionId; import marquez.common.models.NamespaceName; import marquez.common.models.RunId; +import marquez.common.models.RunState; import marquez.db.BaseDao; import marquez.db.DatasetDao; import marquez.db.DatasetVersionDao; @@ -37,11 +38,13 @@ import marquez.db.models.JobRow; import marquez.db.models.RunArgsRow; import marquez.db.models.RunRow; +import marquez.db.models.RunStateRow; import marquez.db.models.UpdateLineageRow; import marquez.service.RunTransitionListener.JobInputUpdate; import marquez.service.RunTransitionListener.JobOutputUpdate; import marquez.service.RunTransitionListener.RunInput; import marquez.service.RunTransitionListener.RunOutput; +import marquez.service.RunTransitionListener.RunTransition; import marquez.service.models.LineageEvent; import marquez.service.models.RunMeta; @@ -91,6 +94,7 @@ public CompletableFuture createAsync(LineageEvent event) { buildJobOutputUpdate(update).ifPresent(runService::notify); } buildJobInputUpdate(update).ifPresent(runService::notify); + buildRunTransition(update).ifPresent(runService::notify); } }); @@ -222,4 +226,15 @@ private DatasetVersionId buildDatasetVersionId(ExtendedDatasetVersionRow ds) { .name(DatasetName.of(ds.getDatasetName())) .build(); } + + private Optional buildRunTransition(UpdateLineageRow record) { + RunId runId = RunId.of(record.getRun().getUuid()); + RunStateRow runStateRow = record.getRunState(); + if (runStateRow == null) { + return Optional.empty(); + } + RunState newState = RunState.valueOf(runStateRow.getState()); + RunState oldState = newState.isStarting() ? null : RunState.RUNNING; + return Optional.of(new RunTransition(runId, oldState, newState)); + } } diff --git a/api/src/test/java/marquez/service/OpenLineageServiceIntegrationTest.java b/api/src/test/java/marquez/service/OpenLineageServiceIntegrationTest.java index 103c749bce..f6eaf51eeb 100644 --- a/api/src/test/java/marquez/service/OpenLineageServiceIntegrationTest.java +++ b/api/src/test/java/marquez/service/OpenLineageServiceIntegrationTest.java @@ -45,6 +45,7 @@ import marquez.jdbi.MarquezJdbiExternalPostgresExtension; import marquez.service.RunTransitionListener.JobInputUpdate; import marquez.service.RunTransitionListener.JobOutputUpdate; +import marquez.service.RunTransitionListener.RunTransition; import marquez.service.models.Dataset; import marquez.service.models.Job; import marquez.service.models.LineageEvent; @@ -79,6 +80,7 @@ public class OpenLineageServiceIntegrationTest { private DatasetVersionDao datasetVersionDao; private ArgumentCaptor runInputListener; private ArgumentCaptor runOutputListener; + private ArgumentCaptor runTransitionListener; private OpenLineageService lineageService; public static String EVENT_REQUIRED_ONLY = "open_lineage/event_required_only.json"; @@ -145,6 +147,8 @@ public void setup(Jdbi jdbi) throws SQLException { doNothing().when(runService).notify(runInputListener.capture()); runOutputListener = ArgumentCaptor.forClass(JobOutputUpdate.class); doNothing().when(runService).notify(runOutputListener.capture()); + runTransitionListener = ArgumentCaptor.forClass(RunTransition.class); + doNothing().when(runService).notify(runTransitionListener.capture()); lineageService = new OpenLineageService(openLineageDao, runService); datasetDao = jdbi.onDemand(DatasetDao.class); @@ -243,6 +247,19 @@ public void testRunListenerOutput(List uris, ExpectedResults expectedResult } } + @ParameterizedTest + @MethodSource("getData") + public void testRunTransition(List uris, ExpectedResults expectedResults) { + initEvents(uris); + + if (expectedResults.inputEventCount > 0) { + Assertions.assertEquals( + uris.size(), + runTransitionListener.getAllValues().size(), + "RunTransition happens once for each run"); + } + } + @ParameterizedTest @MethodSource({"getData"}) public void serviceCalls(List uris, ExpectedResults expectedResults) { From e61fe48ecabe9300f6a8f2899734670ba93bd201 Mon Sep 17 00:00:00 2001 From: Michael Collado <40346148+collado-mike@users.noreply.github.com> Date: Mon, 19 Sep 2022 12:09:32 -0700 Subject: [PATCH 02/25] Add fix and tests for handling Airflow dags with dots and task groups (#2126) Signed-off-by: Michael Collado Signed-off-by: Michael Collado Co-authored-by: Willy Lulciuc --- .../main/java/marquez/db/OpenLineageDao.java | 15 +- .../marquez/OpenLineageIntegrationTest.java | 174 ++++++++++++++++-- 2 files changed, 172 insertions(+), 17 deletions(-) diff --git a/api/src/main/java/marquez/db/OpenLineageDao.java b/api/src/main/java/marquez/db/OpenLineageDao.java index 6b09f4142f..75ce7f4b70 100644 --- a/api/src/main/java/marquez/db/OpenLineageDao.java +++ b/api/src/main/java/marquez/db/OpenLineageDao.java @@ -64,8 +64,8 @@ @RegisterRowMapper(LineageEventMapper.class) public interface OpenLineageDao extends BaseDao { - public String DEFAULT_SOURCE_NAME = "default"; - public String DEFAULT_NAMESPACE_OWNER = "anonymous"; + String DEFAULT_SOURCE_NAME = "default"; + String DEFAULT_NAMESPACE_OWNER = "anonymous"; @SqlUpdate( "INSERT INTO lineage_events (" @@ -370,7 +370,10 @@ private JobRow findParentJobRow( .findJobRowByRunUuid(uuid) .map( j -> { - String parentJobName = Utils.parseParentJobName(facet.getJob().getName()); + String parentJobName = + facet.getJob().getName().equals(event.getJob().getName()) + ? Utils.parseParentJobName(facet.getJob().getName()) + : facet.getJob().getName(); if (j.getNamespaceName().equals(facet.getJob().getNamespace()) && j.getName().equals(parentJobName)) { return j; @@ -432,6 +435,10 @@ private JobRow createParentJobRunRecord( PGobject inputs) { Instant now = event.getEventTime().withZoneSameInstant(ZoneId.of("UTC")).toInstant(); Logger log = LoggerFactory.getLogger(OpenLineageDao.class); + String parentJobName = + facet.getJob().getName().equals(event.getJob().getName()) + ? Utils.parseParentJobName(facet.getJob().getName()) + : facet.getJob().getName(); JobRow newParentJobRow = createJobDao() .upsertJob( @@ -440,7 +447,7 @@ private JobRow createParentJobRunRecord( now, namespace.getUuid(), namespace.getName(), - Utils.parseParentJobName(facet.getJob().getName()), + parentJobName, null, jobContext.getUuid(), location, diff --git a/api/src/test/java/marquez/OpenLineageIntegrationTest.java b/api/src/test/java/marquez/OpenLineageIntegrationTest.java index 3bf575c07a..4233ebe603 100644 --- a/api/src/test/java/marquez/OpenLineageIntegrationTest.java +++ b/api/src/test/java/marquez/OpenLineageIntegrationTest.java @@ -199,11 +199,127 @@ public void testOpenLineageJobHierarchyAirflowIntegration() String dagName = "the_dag"; RunEvent airflowTask1 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); + + RunEvent airflowTask2 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); + + CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); + future.get(5, TimeUnit.SECONDS); + + Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name); + assertThat(job) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name)) + .hasFieldOrPropertyWithValue("parentJobName", dagName); + + Job parentJob = client.getJob(NAMESPACE_NAME, dagName); + assertThat(parentJob) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName)) + .hasFieldOrPropertyWithValue("parentJobName", null); + List runsList = client.listRuns(NAMESPACE_NAME, dagName); + assertThat(runsList).isNotEmpty().hasSize(1); + } + + @Test + public void testOpenLineageJobHierarchyAirflowIntegrationWithDagNameWithDot() + throws ExecutionException, InterruptedException, TimeoutException { + OpenLineage ol = new OpenLineage(URI.create("http://openlineage.test.com/")); + ZonedDateTime startOfHour = + Instant.now() + .atZone(LineageTestUtils.LOCAL_ZONE) + .with(ChronoField.MINUTE_OF_HOUR, 0) + .with(ChronoField.SECOND_OF_MINUTE, 0); + ZonedDateTime endOfHour = startOfHour.plusHours(1); + String airflowParentRunId = UUID.randomUUID().toString(); + String task1Name = "task1"; + String task2Name = "task2"; + String dagName = "the.dag"; + RunEvent airflowTask1 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); + + RunEvent airflowTask2 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); + + CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); + future.get(5, TimeUnit.SECONDS); + + Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name); + assertThat(job) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name)) + .hasFieldOrPropertyWithValue("parentJobName", dagName); + + Job parentJob = client.getJob(NAMESPACE_NAME, dagName); + assertThat(parentJob) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName)) + .hasFieldOrPropertyWithValue("parentJobName", null); + List runsList = client.listRuns(NAMESPACE_NAME, dagName); + assertThat(runsList).isNotEmpty().hasSize(1); + } + + @Test + public void testOpenLineageJobHierarchyAirflowIntegrationWithTaskGroup() + throws ExecutionException, InterruptedException, TimeoutException { + OpenLineage ol = new OpenLineage(URI.create("http://openlineage.test.com/")); + ZonedDateTime startOfHour = + Instant.now() + .atZone(LineageTestUtils.LOCAL_ZONE) + .with(ChronoField.MINUTE_OF_HOUR, 0) + .with(ChronoField.SECOND_OF_MINUTE, 0); + ZonedDateTime endOfHour = startOfHour.plusHours(1); + String airflowParentRunId = UUID.randomUUID().toString(); + String task1Name = "task_group.task1"; + String task2Name = "task_group.task2"; + String dagName = "dag_with_task_group"; + RunEvent airflowTask1 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); RunEvent airflowTask2 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task2Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); future.get(5, TimeUnit.SECONDS); @@ -242,13 +358,27 @@ public void testOpenLineageJobHierarchyOldAirflowIntegration() String task1Name = "task1"; String task2Name = "task2"; String dagName = "the_dag"; + + // the old integration also used the fully qualified task name as the parent job name RunEvent airflowTask1 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName + "." + task1Name, + dagName + "." + task1Name, + NAMESPACE_NAME); RunEvent airflowTask2 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task2Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName + "." + task2Name, + dagName + "." + task2Name, + NAMESPACE_NAME); CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); future.get(5, TimeUnit.SECONDS); @@ -291,12 +421,24 @@ public void testOpenLineageJobHierarchyAirflowIntegrationConflictingRunUuid() // two dag runs with different namespaces - should result in two distinct jobs RunEvent airflowTask1 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); String secondNamespace = "another_namespace"; RunEvent airflowTask2 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, secondNamespace); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + secondNamespace); CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); future.get(5, TimeUnit.SECONDS); @@ -332,7 +474,13 @@ public void testOpenLineageJobHierarchySparkAndAirflow() String dagName = "the_dag"; RunEvent airflowTask1 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); RunEvent sparkTask = createRunEvent( @@ -340,8 +488,8 @@ public void testOpenLineageJobHierarchySparkAndAirflow() startOfHour, endOfHour, airflowTask1.getRun().getRunId().toString(), - sparkTaskName, dagName + "." + task1Name, + dagName + "." + task1Name + "." + sparkTaskName, Optional.empty(), NAMESPACE_NAME); @@ -609,8 +757,8 @@ private RunEvent createAirflowRunEvent( ZonedDateTime startOfHour, ZonedDateTime endOfHour, String airflowParentRunId, - String taskName, String dagName, + String taskName, String namespace) { RunFacet airflowVersionFacet = ol.newRunFacet(); airflowVersionFacet @@ -622,8 +770,8 @@ private RunEvent createAirflowRunEvent( startOfHour, endOfHour, airflowParentRunId, - taskName, dagName, + taskName, Optional.of(airflowVersionFacet), namespace); } @@ -634,8 +782,8 @@ private RunEvent createRunEvent( ZonedDateTime startOfHour, ZonedDateTime endOfHour, String airflowParentRunId, - String taskName, String dagName, + String taskName, Optional airflowVersionFacet, String namespace) { // The Java SDK requires parent run ids to be a UUID, but the python SDK doesn't. In order to @@ -650,7 +798,7 @@ private RunEvent createRunEvent( "run", ImmutableMap.of("runId", airflowParentRunId), "job", - ImmutableMap.of("namespace", namespace, "name", dagName + "." + taskName))); + ImmutableMap.of("namespace", namespace, "name", dagName))); RunFacetsBuilder runFacetBuilder = ol.newRunFacetsBuilder() .nominalTime(ol.newNominalTimeRunFacet(startOfHour, endOfHour)) @@ -663,7 +811,7 @@ private RunEvent createRunEvent( .job( ol.newJob( namespace, - dagName + "." + taskName, + taskName, ol.newJobFacetsBuilder() .documentation(ol.newDocumentationJobFacet("the job docs")) .sql(ol.newSQLJobFacet("SELECT * FROM the_table")) From b608fae68114a9d7a20546d60f8efba3b5b26556 Mon Sep 17 00:00:00 2001 From: Willy Lulciuc Date: Tue, 20 Sep 2022 15:33:49 -0700 Subject: [PATCH 03/25] Bug/version bump in docker up.sh (#2129) * Bump version in .env.example Signed-off-by: wslulciuc * Remove version bump of docker/up.sh in new-version.sh Signed-off-by: wslulciuc * Use VERSION in docker/up.sh Signed-off-by: wslulciuc * Add version bump via VERSION Signed-off-by: wslulciuc * continued: Add version bump via VERSION Signed-off-by: wslulciuc * Fix version bump for docker/up.sh and .env.example Signed-off-by: wslulciuc * Revert version bump for psql Signed-off-by: wslulciuc Signed-off-by: wslulciuc --- .env.example | 2 +- chart/values.yaml | 2 +- docker/up.sh | 11 ++++++----- new-version.sh | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index 106ad5d23a..d89c8f268f 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ API_PORT=5000 API_ADMIN_PORT=5001 WEB_PORT=3000 -TAG=0.19.0 +TAG=0.26.0 diff --git a/chart/values.yaml b/chart/values.yaml index 334e6f14b5..14e9269be0 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -107,7 +107,7 @@ postgresql: ## @param image.tag PostgreSQL image tag (immutable tags are recommended) ## image: - tag: 0.26.0 + tag: 12.1.0 ## Authentication parameters ## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#setting-the-root-password-on-first-run ## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#creating-a-database-on-first-run diff --git a/docker/up.sh b/docker/up.sh index facd6533a3..66df5222af 100755 --- a/docker/up.sh +++ b/docker/up.sh @@ -5,7 +5,8 @@ set -e -SCRIPTDIR=$(dirname $0) +VERSION=0.26.0 +DOCKER_DIR=$(dirname $0) title() { echo -e "\033[1m${1}\033[0m" @@ -35,7 +36,7 @@ usage() { echo " -a, --api-port int api port (default: 5000)" echo " -m, --api-admin-port int api admin port (default: 5001)" echo " -w, --web-port int web port (default: 3000)" - echo " -t, --tag string image tag (default: latest)" + echo " -t, --tag string image tag (default: ${VERSION})" echo title "FLAGS:" echo " -b, --build build images from source" @@ -55,7 +56,7 @@ args="-V --force-recreate --remove-orphans" API_PORT=5000 API_ADMIN_PORT=5001 WEB_PORT=3000 -TAG=0.19.0 +TAG=${VERSION} while [ $# -gt 0 ]; do case $1 in -a|'--api-port') @@ -107,6 +108,6 @@ if [[ "${SEED}" = "true" ]]; then compose_files+=" -f docker-compose.seed.yml" fi -$SCRIPTDIR/volumes.sh marquez +${DOCKER_DIR}/volumes.sh marquez -API_PORT=${API_PORT} API_ADMIN_PORT=${API_ADMIN_PORT} WEB_PORT=${WEB_PORT} TAG="${TAG}" docker-compose $compose_files up $args +API_PORT=${API_PORT} API_ADMIN_PORT=${API_ADMIN_PORT} WEB_PORT=${WEB_PORT} TAG=${TAG} docker-compose $compose_files up $args diff --git a/new-version.sh b/new-version.sh index 6b01ec06b6..c805a9c022 100755 --- a/new-version.sh +++ b/new-version.sh @@ -130,8 +130,8 @@ sed -i "" "s/^version:.*/version: ${RELEASE_VERSION}/g" ./chart/Chart.yaml sed -i "" "s/tag:.*/tag: ${RELEASE_VERSION}/g" ./chart/values.yaml # (3) Bump version in scripts -sed -i "" "s/TAG=\d.*/TAG=${RELEASE_VERSION}/g" ./docker/up.sh -sed -i "" "s/TAG=\d.*/TAG=${RELEASE_VERSION}/g" .env.example +sed -i "" "s/VERSION=.*/VERSION=${RELEASE_VERSION}/g" ./docker/up.sh +sed -i "" "s/TAG=.*/TAG=${RELEASE_VERSION}/g" .env.example # (4) Bump version in docs sed -i "" "s/^ version:.*/ version: ${RELEASE_VERSION}/g" ./spec/openapi.yml From 0aaa489de2d2d5c49fecdcee6d1818003b5e6ad8 Mon Sep 17 00:00:00 2001 From: Michael Collado <40346148+collado-mike@users.noreply.github.com> Date: Wed, 21 Sep 2022 12:34:34 -0700 Subject: [PATCH 04/25] =?UTF-8?q?Add=20parentRun=20alias=20to=20LineageEve?= =?UTF-8?q?nt=20RunFacet=20to=20support=20older=20OpenLin=E2=80=A6=20(#213?= =?UTF-8?q?0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add parentRun alias to LineageEvent RunFacet to support older OpenLineage events Signed-off-by: Michael Collado * Add changelog update Signed-off-by: Michael Collado Signed-off-by: Michael Collado --- CHANGELOG.md | 2 + .../marquez/service/models/LineageEvent.java | 7 +- .../marquez/OpenLineageIntegrationTest.java | 134 ++++++++++++++++++ 3 files changed, 142 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64229b65c3..69e267abb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.26.0...HEAD) +### Fixed +* Add support for `parentRun` facet as reported by older Airflow OpenLineage versions [@collado-mike](https://github.com/collado-mike) ## [0.26.0](https://github.com/MarquezProject/marquez/compare/0.25.0...0.26.0) - 2022-09-15 ### Added diff --git a/api/src/main/java/marquez/service/models/LineageEvent.java b/api/src/main/java/marquez/service/models/LineageEvent.java index d1b694391e..1de50c2b7b 100644 --- a/api/src/main/java/marquez/service/models/LineageEvent.java +++ b/api/src/main/java/marquez/service/models/LineageEvent.java @@ -5,6 +5,7 @@ package marquez.service.models; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -71,7 +72,11 @@ public static class Run extends BaseJsonModel { public static class RunFacet { @Valid private NominalTimeRunFacet nominalTime; - @Valid private ParentRunFacet parent; + + @JsonAlias( + "parentRun") // the Airflow integration previously reported parentRun instead of parent + @Valid + private ParentRunFacet parent; @Builder.Default @JsonIgnore private Map additional = new LinkedHashMap<>(); diff --git a/api/src/test/java/marquez/OpenLineageIntegrationTest.java b/api/src/test/java/marquez/OpenLineageIntegrationTest.java index 4233ebe603..8e3f8ce0a3 100644 --- a/api/src/test/java/marquez/OpenLineageIntegrationTest.java +++ b/api/src/test/java/marquez/OpenLineageIntegrationTest.java @@ -31,6 +31,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -235,6 +236,139 @@ public void testOpenLineageJobHierarchyAirflowIntegration() assertThat(runsList).isNotEmpty().hasSize(1); } + @Test + public void testOpenLineageJobHierarchyAirflowIntegrationWithParentRunFacet() + throws ExecutionException, InterruptedException, TimeoutException { + OpenLineage ol = new OpenLineage(URI.create("http://openlineage.test.com/")); + ZonedDateTime startOfHour = + Instant.now() + .atZone(LineageTestUtils.LOCAL_ZONE) + .with(ChronoField.MINUTE_OF_HOUR, 0) + .with(ChronoField.SECOND_OF_MINUTE, 0); + ZonedDateTime endOfHour = startOfHour.plusHours(1); + String airflowParentRunId = UUID.randomUUID().toString(); + String task1Name = "task1"; + String task2Name = "task2"; + String dagName = "the_dag"; + RunEvent airflowTask1 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); + + // the older airflow integration reported parentRun instead of parent. We support this as an + // alias for compatibility + RunFacet parent = airflowTask1.getRun().getFacets().getAdditionalProperties().remove("parent"); + airflowTask1.getRun().getFacets().getAdditionalProperties().put("parentRun", parent); + + RunEvent airflowTask2 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); + parent = airflowTask2.getRun().getFacets().getAdditionalProperties().remove("parent"); + airflowTask2.getRun().getFacets().getAdditionalProperties().put("parentRun", parent); + + CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); + future.get(5, TimeUnit.SECONDS); + + Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name); + assertThat(job) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name)) + .hasFieldOrPropertyWithValue("parentJobName", dagName); + + Job parentJob = client.getJob(NAMESPACE_NAME, dagName); + assertThat(parentJob) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName)) + .hasFieldOrPropertyWithValue("parentJobName", null); + List runsList = client.listRuns(NAMESPACE_NAME, dagName); + assertThat(runsList).isNotEmpty().hasSize(1); + } + + @Test + public void testOpenLineageJobHierarchyAirflowIntegrationWithParentAndParentRunFacet() + throws ExecutionException, InterruptedException, TimeoutException { + OpenLineage ol = new OpenLineage(URI.create("http://openlineage.test.com/")); + ZonedDateTime startOfHour = + Instant.now() + .atZone(LineageTestUtils.LOCAL_ZONE) + .with(ChronoField.MINUTE_OF_HOUR, 0) + .with(ChronoField.SECOND_OF_MINUTE, 0); + ZonedDateTime endOfHour = startOfHour.plusHours(1); + String airflowParentRunId = UUID.randomUUID().toString(); + String task1Name = "task1"; + String task2Name = "task2"; + String dagName = "the_dag"; + RunEvent airflowTask1 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); + + // the older airflow integration reported parentRun instead of parent. The new integration + // reports both. They are the same in the airflow integration, but this test verifies we handle + // the "parentRun" field first. + // It would be preferable to prioritize the "parent" field, but it seems Jackson prefers the + // alias first. + RunFacet parent = airflowTask1.getRun().getFacets().getAdditionalProperties().get("parent"); + RunFacet newParent = ol.newRunFacet(); + Map runFacetProps = newParent.getAdditionalProperties(); + runFacetProps.put("run", parent.getAdditionalProperties().get("run")); + runFacetProps.put( + "job", ImmutableMap.of("name", "a_new_dag", "namespace", "incorrect_namespace")); + airflowTask1.getRun().getFacets().getAdditionalProperties().put("parentRun", parent); + airflowTask1.getRun().getFacets().getAdditionalProperties().put("parent", newParent); + + RunEvent airflowTask2 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); + parent = airflowTask2.getRun().getFacets().getAdditionalProperties().get("parent"); + newParent = ol.newRunFacet(); + runFacetProps = newParent.getAdditionalProperties(); + runFacetProps.put("run", parent.getAdditionalProperties().get("run")); + runFacetProps.put( + "job", ImmutableMap.of("name", "a_new_dag", "namespace", "incorrect_namespace")); + airflowTask2.getRun().getFacets().getAdditionalProperties().put("parentRun", parent); + airflowTask2.getRun().getFacets().getAdditionalProperties().put("parent", newParent); + + CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); + future.get(5, TimeUnit.SECONDS); + + Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name); + assertThat(job) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name)) + .hasFieldOrPropertyWithValue("parentJobName", dagName); + + Job parentJob = client.getJob(NAMESPACE_NAME, dagName); + assertThat(parentJob) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName)) + .hasFieldOrPropertyWithValue("parentJobName", null); + List runsList = client.listRuns(NAMESPACE_NAME, dagName); + assertThat(runsList).isNotEmpty().hasSize(1); + } + @Test public void testOpenLineageJobHierarchyAirflowIntegrationWithDagNameWithDot() throws ExecutionException, InterruptedException, TimeoutException { From 9f0280abd0ec5179983953b26b342ffc7f859a71 Mon Sep 17 00:00:00 2001 From: Willy Lulciuc Date: Fri, 23 Sep 2022 13:24:26 -0700 Subject: [PATCH 05/25] Fix `codecov` in CI (#2133) * Use codecov/upload Signed-off-by: wslulciuc * continued: Use codecov/upload Signed-off-by: wslulciuc Signed-off-by: wslulciuc --- .circleci/config.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ea43b23a18..8fcb3e11d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,10 @@ only-on-release: &only-on-release branches: ignore: /.*/ +orbs: + # https://circleci.com/orbs/registry/orb/codecov/codecov + codecov: codecov/codecov@3.2.3 + jobs: build-api: working_directory: ~/marquez @@ -32,7 +36,7 @@ jobs: - run: ./gradlew --no-daemon --stacktrace api:javadoc - run: ./gradlew --no-daemon --stacktrace api:build - run: ./gradlew --no-daemon api:jacocoTestReport - - run: bash <(curl -s https://codecov.io/bash) + - codecov/upload - store_test_results: path: api/build/test-results/test - store_artifacts: @@ -99,7 +103,7 @@ jobs: - run: ./gradlew --no-daemon --stacktrace clients:java:javadoc - run: ./gradlew --no-daemon --stacktrace clients:java:build - run: ./gradlew --no-daemon clients:java:jacocoTestReport - - run: bash <(curl -s https://codecov.io/bash) + - codecov/upload - store_test_results: path: clients/java/build/test-results/test - store_artifacts: @@ -119,7 +123,7 @@ jobs: - run: pip install -e .[dev] - run: flake8 - run: pytest --cov=marquez_python tests/ - - run: bash <(curl -s https://codecov.io/bash) + - codecov/upload build-client-python: working_directory: ~/marquez/clients/python From 338a681bec410ad6dd778fe01defedb514455b6e Mon Sep 17 00:00:00 2001 From: Howard Yoo <32691630+howardyoo@users.noreply.github.com> Date: Fri, 23 Sep 2022 15:37:32 -0500 Subject: [PATCH 06/25] Fixes issue: #2131 (#2132) Signed-off-by: howardyoo Signed-off-by: howardyoo Co-authored-by: Willy Lulciuc --- spec/openapi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/openapi.yml b/spec/openapi.yml index ba4e3af143..111c61655f 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -339,14 +339,14 @@ paths: summary: Soft deletes job. description: Soft deletes job. It will be un-deleted if new OpenLineage event containing this job comes. tags: - - Datasets + - Jobs responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/Dataset' + $ref: '#/components/schemas/Job' /namespaces/{namespace}/jobs: parameters: From ffcf1e896598d3a086bf439afdac6117e0f02a52 Mon Sep 17 00:00:00 2001 From: Michael Robinson <68482867+merobi-hub@users.noreply.github.com> Date: Fri, 23 Sep 2022 16:45:06 -0400 Subject: [PATCH 07/25] [Docs] Add Github workflow to check for header in source code (#2128) * add github workflow to check for header and remove header for testing Signed-off-by: Michael Robinson * fix on section of headerchecker.yml for initial commits Signed-off-by: Michael Robinson * replace header for testing Signed-off-by: Michael Robinson Signed-off-by: Michael Robinson --- .github/workflows/headerchecker.yml | 47 +++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/headerchecker.yml diff --git a/.github/workflows/headerchecker.yml b/.github/workflows/headerchecker.yml new file mode 100644 index 0000000000..5c232a43cc --- /dev/null +++ b/.github/workflows/headerchecker.yml @@ -0,0 +1,47 @@ +--- +name: Check for Headers in Source Code + +on: + push: + branches: ['renovate/**'] + pull_request: + branches: [main] + +jobs: + build: + name: Check for Headers + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - id: files + uses: jitterbit/get-changed-files@v1 + with: + format: 'json' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check for headers + run: | + ok=1 + readarray -t files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.all }}')" + for file in ${files[@]}; do + if [[ ($file == *".java") ]]; then + if ! grep -q Copyright "$file"; then + ok=0 + echo "Copyright header not found in $file" + fi + fi + done + if [[ $ok == 0 ]]; then + exit 1 + else + GREEN="\e[32m" + echo -e "${GREEN}All changed & added files have been scanned. Result: no headers are missing.${ENDCOLOR}" + fi \ No newline at end of file From b446e4e4c87ed2468fb5ca3702541182a21eb24a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:01:09 -0700 Subject: [PATCH 08/25] chore(deps): update azure/setup-helm action to v2.2 (#2103) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/test-chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-chart.yaml b/.github/workflows/test-chart.yaml index 36584d84ee..fc52ed1886 100644 --- a/.github/workflows/test-chart.yaml +++ b/.github/workflows/test-chart.yaml @@ -16,7 +16,7 @@ jobs: fetch-depth: 0 - name: Setup Helm - uses: azure/setup-helm@v2.1 + uses: azure/setup-helm@v2.2 - name: Setup Python uses: actions/setup-python@v3 From ecda00756e891c3c9ed5706a931446008933be57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:02:34 -0700 Subject: [PATCH 09/25] Bump moment from 2.29.2 to 2.29.4 in /web (#2056) Bumps [moment](https://github.com/moment/moment) from 2.29.2 to 2.29.4. - [Release notes](https://github.com/moment/moment/releases) - [Changelog](https://github.com/moment/moment/blob/develop/CHANGELOG.md) - [Commits](https://github.com/moment/moment/compare/2.29.2...2.29.4) --- updated-dependencies: - dependency-name: moment dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- web/package-lock.json | 14 +++++++------- web/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index b9c5a8c20c..88af3e021e 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -39,7 +39,7 @@ "dagre": "^0.8.5", "http-proxy-middleware": "^0.20.0", "lodash": "^4.17.21", - "moment": "^2.29.2", + "moment": "^2.29.4", "postcss-loader": "^3.0.0", "postcss-modules-values": "^2.0.0", "react": "^16.8.0", @@ -11889,9 +11889,9 @@ } }, "node_modules/moment": { - "version": "2.29.2", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz", - "integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==", + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==", "engines": { "node": "*" } @@ -25783,9 +25783,9 @@ } }, "moment": { - "version": "2.29.2", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz", - "integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==" + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==" }, "moo": { "version": "0.4.3", diff --git a/web/package.json b/web/package.json index 36816676f0..b4a17586b3 100644 --- a/web/package.json +++ b/web/package.json @@ -47,7 +47,7 @@ "dagre": "^0.8.5", "http-proxy-middleware": "^0.20.0", "lodash": "^4.17.21", - "moment": "^2.29.2", + "moment": "^2.29.4", "postcss-loader": "^3.0.0", "postcss-modules-values": "^2.0.0", "react": "^16.8.0", From 5e4b223d2264e389ee254e9926fa840225cbaee1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:03:29 -0700 Subject: [PATCH 10/25] chore(deps): update dependency gradle to v7.5.1 (#2104) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Willy Lulciuc --- gradle/wrapper/gradle-wrapper.jar | Bin 59821 -> 60756 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 6 ++++++ gradlew.bat | 14 ++++++++------ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 41d9927a4d4fb3f96a785543079b8df6723c946b..249e5832f090a2944b7473328c07c9755baa3196 100644 GIT binary patch delta 10197 zcmaKS1ymhDwk=#NxVyW%y9U<)A-Dv)xI0|j{UX8L-JRg>5ZnnKAh;%chM6~S-g^K4 z>eZ{yK4;gd>gwvXs=Id8Jk-J}R4pT911;+{Jp9@aiz6!p1Oz9z&_kGLA%J5%3Ih@0 zQ|U}%$)3u|G`jIfPzMVfcWs?jV2BO^*3+q2><~>3j+Z`^Z%=;19VWg0XndJ zwJ~;f4$;t6pBKaWn}UNO-wLCFHBd^1)^v%$P)fJk1PbK5<;Z1K&>k~MUod6d%@Bq9 z>(44uiaK&sdhwTTxFJvC$JDnl;f}*Q-^01T508(8{+!WyquuyB7R!d!J)8Ni0p!cV6$CHsLLy6}7C zYv_$eD;)@L)tLj0GkGpBoa727hs%wH$>EhfuFy{_8Q8@1HI%ZAjlpX$ob{=%g6`Ox zLzM!d^zy`VV1dT9U9(^}YvlTO9Bf8v^wMK37`4wFNFzW?HWDY(U(k6@tp(crHD)X5>8S-# zW1qgdaZa*Sh6i%60e1+hty}34dD%vKgb?QmQiZ=-j+isA4={V_*R$oGN#j|#ia@n6 zuZx4e2Xx?^lUwYFn2&Tmbx0qA3Z8;y+zKoeQu;~k~FZGy!FU_TFxYd!Ck;5QvMx9gj5fI2@BLNp~Ps@ zf@k<&Q2GS5Ia9?_D?v~$I%_CLA4x~eiKIZ>9w^c#r|vB?wXxZ(vXd*vH(Fd%Me8p( z=_0)k=iRh%8i`FYRF>E97uOFTBfajv{IOz(7CU zv0Gd84+o&ciHlVtY)wn6yhZTQQO*4Mvc#dxa>h}82mEKKy7arOqU$enb9sgh#E=Lq zU;_RVm{)30{bw+|056%jMVcZRGEBSJ+JZ@jH#~DvaDQm92^TyUq=bY*+AkEakpK>8 zB{)CkK48&nE5AzTqT;WysOG|!y}5fshxR8Ek(^H6i>|Fd&wu?c&Q@N9ZrJ=?ABHI! z`*z8D`w=~AJ!P-9M=T}f`;76$qZRllB&8#9WgbuO$P7lVqdX1=g*t=7z6!0AQ^ux_ z9rcfUv^t}o_l-ZE+TqvqFsA*~W<^78!k;~!i8(eS+(+@u8FxK+Q7;mHZ<1}|4m<}vh@p`t%|@eM_J(P% zI>M7C)Ir{l|J;$G_EGGEhbP4?6{sYzMqBv+x95N&YWFH6UcE@b}B?q)G*4<4mR@sy1#vPnLMK51tb#ED(8TA1nE zYfhK7bo1!R5WJF$5Y?zG21)6+_(_5oSX9sGIW;(O&S?Rh(nydNQYzKjjJ54aDJ-1F zrJ=np8LsN?%?Rt7f~3aAX!2E{`fh_pb?2(;HOB3W+I*~A>W%iY+v45+^e$cE10fA} zXPvw9=Bd+(;+!rl)pkYj0HGB}+3Z!Mr;zr%gz~c-hFMv8b2VRE2R$8V=_XE zq$3=|Yg05(fmwrJ)QK2ptB4no`Y8Dg_vK2QDc6-6sXRQ5k78-+cPi-fH}vpgs|Ive zE=m*XNVs?EWgiNI!5AcD*3QMW)R`EqT!f0e1%hERO&?AT7HWnSf5@#AR{OGuXG3Zb zCnVWg7h|61lGV3k+>L<#d>)InG>ETn1DbOHCfztqzQ_fBiaUt@q6VMy={Fe-w#~2- z0?*f|z$zgjI9>+JVICObBaK=pU}AEOd@q(8d?j7zQFD@=6t`|KmolTr2MfBI$;EGh zD%W0cA_d#V6Lb$us5yIG(|d>r-QleC4;%hEu5W9hyY zY#+ESY&v`8(&mC~?*|e5WEhC!YU2>m_}`K+q9)a(d$bsS<=YkyZGp}YA%TXw>@abA zS_poVPoN+?<6?DAuCNt&5SHV(hp56PJ})swwVFZFXM->F zc|0c8<$H_OV%DR|y7e+s$12@Ac8SUClPg8_O9sTUjpv%6Jsn5vsZCg>wL+db4c+{+ zsg<#wOuV4jeOq`veckdi-1`dz;gvL)bZeH|D*x=8UwRU5&8W1@l>3$)8WzET0%;1J zM3(X<7tKK&9~kWRI{&FmwY5Gg!b5f4kI_vSm)H1#>l6M+OiReDXC{kPy!`%Ecq-+3yZTk=<` zm)pE6xum5q0Qkd#iny0Q-S}@I0;mDhxf>sX)Oiv)FdsAMnpx%oe8OQ`m%Xeozdzx!C1rQR>m1c_}+J4x)K}k{G zo68;oGG&Ox7w^-m7{g4a7NJu-B|~M;oIH~~#`RyUNm##feZH;E?pf}nshmoiIY52n z%pc%lnU4Q#C=RUz)RU6}E_j4#)jh<&a%JyJj$Fufc#&COaxFHtl}zJUGNLBu3~_@1 zn9F^JO9);Duxo&i@>X(kbYga1i>6p1fca8FzQ0>((Lb-aPUbC*d~a03V$y;*RBY!R ziEJ2IF^FjrvO}0Uy{cMn%u<+P5U!UO>pm9#ZYL5i6|xSC+np7IH$GfXs&uI;y4as@ z&AzJh>(S2?3PKKgab3Z(`xbx(C#46XIvVcW8eG_DjT~}Yz_8PWZ`uf6^Xr=vkvL_` zqmvfgJL+Zc`;iq~iP?%@G7}~fal-zqxa0yNyHBJJ5M)9bI>7S_cg?Ya&p(I)C5Ef4 zZ>YAF6x|U=?ec?g*|f2g5Tw3PgxaM_bi_5Az9MO$;_Byw(2d}2%-|bg4ShdQ;)Z|M z4K|tFv)qx*kKGKoyh!DQY<{n&UmAChq@DJrQP>EY7g1JF(ih*D8wCVWyQ z5Jj^|-NVFSh5T0vd1>hUvPV6?=`90^_)t(L9)XOW7jeP45NyA2lzOn&QAPTl&d#6P zSv%36uaN(9i9WlpcH#}rmiP#=L0q(dfhdxvFVaOwM;pY;KvNQ9wMyUKs6{d}29DZQ z{H3&Sosr6)9Z+C>Q5)iHSW~gGoWGgK-0;k~&dyr-bA3O|3PCNzgC?UKS_B=^i8Ri^ zd_*_qI4B07Cayq|p4{`U_E_P=K`N_~{F|+-+`sCgcNxs`%X!$=(?l2aAW}0M=~COb zf19oe^iuAUuDEf)4tgv<=WRPpK@IjToNNC*#&Ykw!)aqWU4h#|U@(cG_=Qx+&xt~a zvCz~Ds3F71dsjNLkfM%TqdVNu=RNMOzh7?b+%hICbFlOAPphrYy>7D-e7{%o_kPFn z;T!?ilE-LcKM0P(GKMseEeW57Vs`=FF}(y@^pQl;rL3fHs8icmA+!6YJt&8 ztSF?%Un35qkv>drkks&BNTJv~xK?vD;aBkp7eIkDYqn+G0%;sT4FcwAoO+vke{8CO z0d76sgg$CannW5T#q`z~L4id)9BCKRU0A!Z-{HpXr)QJrd9@iJB+l32Ql)Z}*v(St zE)Vp=BB=DDB4Pr}B(UHNe31<@!6d{U?XDoxJ@S)9QM)2L%SA0x^~^fb=bdsBy!uh& zU?M_^kvnt%FZzm+>~bEH{2o?v&Iogs`1t-b+Ml`J!ZPS(46YQJKxWE81O$HE5w;** z|8zM%bp`M7J8)4;%DqH`wVTmM0V@D}xd%tRE3_6>ioMJxyi5Hkb>85muF81&EY!73ei zA3e<#ug||EZJ=1GLXNJ)A z791&ge#lF;GVX6IU?iw0jX^1bYaU?+x{zPlpyX6zijyn*nEdZ$fxxkl!a-~*P3bkf zPd*pzu~3GBYkR_>ET`5UM^>>zTV>5m>)f=az{d0sg6a8VzUtXy$ZS?h#Gk-CA?7)c zI%Vu9DN6XSDQn6;?n9`>l$q&>s?K)R8*OsmI+$L_m z_~E`}w694Z*`Xk3Ne=497Si~=RWRqCM?6=88smrxle#s*W znwhTRsMRmg?37GLJ-)%nDZA7r$YG849j8mJWir1bWBy& zZPneYojSbooC8U@tkO`bWx4%E5*;p#Q^1^S3lsfy7(6A{jL0`A__0vm?>xC%1y8_m z57FfWr^@YG2I1K7MGYuYd>JC}@sT2n^rkrY3w%~$J$Y~HSoOHn?zpR$ zjLj_bq@Yj8kd~DXHh30KVbz@K)0S;hPKm+S&-o%IG+@x@MEcrxW2KFh;z^4dJDZix zGRGe&lQD$p)0JVF4NRgGYuh0bYLy)BCy~sbS3^b3 zHixT<%-Vwbht|25T{3^Hk;qZ^3s!OOgljHs+EIf~C%=_>R5%vQI4mQR9qOXThMXlU zS|oSH>0PjnCakb*js2{ObN`}%HYsT6=%(xA| znpUtG_TJ08kHgm5l@G|t?4E3tG2fq?wNtIp*Vqrb{9@bo^~Rx7+J&OnayrX`LDcF~ zd@0m0ZJ#Z@=T>4kTa5e2FjI&5c(F7S{gnRPoGpu9eIqrtSvnT_tk$8T)r%YwZw!gK zj*k@cG)V&@t+mtDi37#>LhVGTfRA^p%x0d#_P|Mktz3*KOoLIqFm`~KGoDDD4OOxe z?}ag_c08u%vu=5Vx=~uoS8Q;}+R2~?Uh|m-+`-2kDo$d6T!nD*hc#dB(*R{LXV=zo z`PJP0V=O!@3l-bw+d`X6(=@fq=4O#ETa8M^fOvO4qja9o3e8ANc9$sI=A4$zUut~w z4+JryRkI{9qWxU1CCMM$@Aj=6)P+z?vqa=UCv_4XyVNoBD{Xb~Oi4cjjhm8fRD!*U z2)zaS;AI78^Wq+5mDInKiMz|z#K`2emQfNH*U;{9^{NqSMVoq?RSo43<8YpJM^+W$ zxy!A5>5Zl16Vi#?nAYywu3w_=KWnd3*QetocWt`3pK67>)ZVwnT3h zbPdD&MZkD?q=-N`MpCCwpM74L+Tr1aa)zJ)8G;(Pg51@U&5W>aNu9rA`bh{vgfE={ zdJ>aKc|2Ayw_bop+dK?Y5$q--WM*+$9&3Q9BBiwU8L<-`T6E?ZC`mT0b}%HR*LPK} z!MCd_Azd{36?Y_>yN{U1w5yrN8q`z(Vh^RnEF+;4b|2+~lfAvPT!`*{MPiDioiix8 zY*GdCwJ{S(5(HId*I%8XF=pHFz<9tAe;!D5$Z(iN#jzSql4sqX5!7Y?q4_%$lH zz8ehZuyl0K=E&gYhlfFWabnSiGty$>md|PpU1VfaC5~kskDnZX&Yu}?-h;OSav=8u z=e3Yq=mi$4A|sB-J00;1d{Sd1+!v0NtU((Nz2;PFFlC}V{@p&4wGcVhU&nI($RAS! zwXn7)?8~1J3*4+VccRSg5JS<(bBhBM&{ELMD4C_NTpvzboH!{Zr*%HP;{UqxI#g&7 zOAqPSW5Qus$8-xtTvD%h{Tw<2!XR(lU54LZG{)Cah*LZbpJkA=PMawg!O>X@&%+5XiyeIf91n2E*hl$k-Y(3iW*E}Mz-h~H~7S9I1I zR#-j`|Hk?$MqFhE4C@=n!hN*o5+M%NxRqP+aLxDdt=wS6rAu6ECK*;AB%Nyg0uyAv zO^DnbVZZo*|Ef{nsYN>cjZC$OHzR_*g%T#oF zCky9HJS;NCi=7(07tQXq?V8I&OA&kPlJ_dfSRdL2bRUt;tA3yKZRMHMXH&#W@$l%-{vQd7y@~i*^qnj^`Z{)V$6@l&!qP_y zg2oOd!Wit#)2A~w-eqw3*Mbe)U?N|q6sXw~E~&$!!@QYX4b@%;3=>)@Z#K^`8~Aki z+LYKJu~Y$;F5%_0aF9$MsbGS9Bz2~VUG@i@3Fi2q(hG^+Ia44LrfSfqtg$4{%qBDM z_9-O#3V+2~W$dW0G)R7l_R_vw(KSkC--u&%Rs^Io&*?R=`)6BN64>6>)`TxyT_(Rd zUn+aIl1mPa#Jse9B3`!T=|e!pIp$(8ZOe0ao?nS7o?oKlj zypC-fMj1DHIDrh1unUI1vp=-Fln;I9e7Jvs3wj*^_1&W|X} zZSL|S|Bb@CV*YC_-T&2!Ht3b6?)d`tHOP?rA;;t#zaXa0Sc;vGnV0BLIf8f-r{QHh z*Zp`4_ItlOR7{u(K+!p_oLDmaAkNag*l4#29F2b_A*0oz0T|#-&f*;c#<`^)(W@gm z#k9k=t%u8<+C1fNUA{Fh7~wgPrEZZ#(6aBI%6bR4RO(e1(ZocjoDek4#MTgZD>1NG zy9~yoZfWYfwe&S-(zk4o6q6o?2*~DOrJ(%5wSnEJMVOKCzHd z=Yhm+HLzoDl{P*Ybro7@sk1!Ez3`hE+&qr7Rw^2glw^M(b(NS2!F|Q!mi|l~lF94o z!QiV)Q{Z>GO5;l1y!$O)=)got;^)%@v#B!ZEVQy1(BJApHr5%Zh&W|gweD+%Ky%CO ztr45vR*y(@*Dg_Qw5v~PJtm^@Lyh*zRuT6~(K+^HWEF{;R#L$vL2!_ndBxCtUvZ(_ zauI7Qq}ERUWjr&XW9SwMbU>*@p)(cuWXCxRK&?ZoOy>2VESII53iPDP64S1pl{NsC zD;@EGPxs&}$W1;P6BB9THF%xfoLX|4?S;cu@$)9OdFst-!A7T{(LXtdNQSx!*GUSIS_lyI`da8>!y_tpJb3Zuf0O*;2y?HCfH z5QT6@nL|%l3&u4;F!~XG9E%1YwF*Fgs5V&uFsx52*iag(?6O|gYCBY3R{qhxT-Etb zq(E%V=MgQnuDGEKOGsmBj9T0-nmI%zys8NSO>gfJT4bP>tI>|ol@ zDt(&SUKrg%cz>AmqtJKEMUM;f47FEOFc%Bbmh~|*#E zDd!Tl(wa)ZZIFwe^*)4>{T+zuRykc3^-=P1aI%0Mh}*x7%SP6wD{_? zisraq`Las#y-6{`y@CU3Ta$tOl|@>4qXcB;1bb)oH9kD6 zKym@d$ zv&PZSSAV1Gwwzqrc?^_1+-ZGY+3_7~a(L+`-WdcJMo>EWZN3%z4y6JyF4NR^urk`c z?osO|J#V}k_6*9*n2?j+`F{B<%?9cdTQyVNm8D}H~T}?HOCXt%r7#2hz97Gx#X%62hyaLbU z_ZepP0<`<;eABrHrJAc!_m?kmu#7j}{empH@iUIEk^jk}^EFwO)vd7NZB=&uk6JG^ zC>xad8X$h|eCAOX&MaX<$tA1~r|hW?-0{t4PkVygTc`yh39c;&efwY(-#;$W)+4Xb z$XFsdG&;@^X`aynAMxsq)J#KZXX!sI@g~YiJdHI~r z$4mj_?S29sIa4c$z)19JmJ;Uj?>Kq=0XuH#k#};I&-6zZ_&>)j>UR0XetRO!-sjF< zd_6b1A2vfi++?>cf}s{@#BvTD|a%{9si7G}T+8ZnwuA z1k8c%lgE<-7f~H`cqgF;qZ|$>R-xNPA$25N1WI3#n%gj}4Ix}vj|e=x)B^roGQpB) zO+^#nO2 zjzJ9kHI6nI5ni&V_#5> z!?<7Qd9{|xwIf4b0bRc;zb}V4>snRg6*wl$Xz`hRDN8laL5tg&+@Dv>U^IjGQ}*=XBnXWrwTy;2nX?<1rkvOs#u(#qJ=A zBy>W`N!?%@Ay=upXFI}%LS9bjw?$h)7Dry0%d}=v0YcCSXf9nnp0tBKT1eqZ-4LU` zyiXglKRX)gtT0VbX1}w0f2ce8{$WH?BQm@$`ua%YP8G@<$n13D#*(Yd5-bHfI8!on zf5q4CPdgJLl;BqIo#>CIkX)G;rh|bzGuz1N%rr+5seP${mEg$;uQ3jC$;TsR&{IX< z;}7j3LnV+xNn^$F1;QarDf6rNYj7He+VsjJk6R@0MAkcwrsq4?(~`GKy|mgkfkd1msc2>%B!HpZ~HOzj}kl|ZF(IqB=D6ZTVcKe=I7)LlAI=!XU?J*i#9VXeKeaG zwx_l@Z(w`)5Cclw`6kQKlS<;_Knj)^Dh2pL`hQo!=GPOMR0iqEtx12ORLpN(KBOm5 zontAH5X5!9WHS_=tJfbACz@Dnkuw|^7t=l&x8yb2a~q|aqE_W&0M|tI7@ilGXqE)MONI8p67OiQGqKEQWw;LGga=ZM1;{pSw1jJK_y$vhY6 ztFrV7-xf>lbeKH1U)j3R=?w*>(Yh~NNEPVmeQ8n}0x01$-o z2Jyjn+sXhgOz>AzcZ zAbJZ@f}MBS0lLKR=IE{z;Fav%tcb+`Yi*!`HTDPqSCsFr>;yt^^&SI2mhKJ8f*%ji zz%JkZGvOn{JFn;)5jf^21AvO-9nRzsg0&CPz;OEn07`CfT@gK4abFBT$Mu?8fCcscmRkK+ zbAVJZ~#_a z{|(FFX}~8d3;DW8zuY9?r#Dt>!aD>} zlYw>D7y#eDy+PLZ&XKIY&Df0hsLDDi(Yrq8O==d30RchrUw8a=Eex>Dd?)3+k=}Q> z-b85lun-V$I}86Vg#l1S@1%=$2BQD5_waAZKQfJ${3{b2SZ#w1u+jMr{dJMvI|Og= zpQ9D={XK|ggbe04zTUd}iF{`GO1dV%zWK~?sM9OM(= zVK9&y4F^w1WFW{$qi|xQk0F`@HG8oLI5|5$j~ci9xTMT69v5KS-Yym--raU5kn2#C z<~5q^Bf0rTXVhctG2%&MG(cUGaz(gC(rcG~>qgO$W6>!#NOVQJ;pIYe-lLy(S=HgI zPh;lkL$l+FfMHItHnw_^bj8}CKM19t(C_2vSrhX2$K@-gFlH};#C?1;kk&U1L%4S~ zR^h%h+O1WE7DI$~dly?-_C7>(!E`~#REJ~Xa7lyrB$T!`&qYV5QreAa^aKr%toUJR zPWh)J3iD`(P6BI5k$oE$us#%!4$>`iH2p-88?WV0M$-K)JDibvA4 zpef%_*txN$Ei3=Lt(BBxZ&mhl|mUz-z*OD1=r9nfN zc5vOMFWpi>K=!$6f{eb?5Ru4M3o;t9xLpry|C%j~`@$f)OFB5+xo8XM8g&US@UU-sB|dAoc20y(F@=-2Ggp_`SWjEb#>IG^@j zuQK}e^>So#W2%|-)~K!+)wdU#6l>w5wnZt2pRL5Dz#~N`*UyC9tYechBTc2`@(OI# zNvcE*+zZZjU-H`QOITK^tZwOyLo)ZCLk>>Wm+flMsr5X{A<|m`Y281n?8H_2Fkz5}X?i%Rfm5s+n`J zDB&->=U+LtOIJ|jdYXjQWSQZFEs>Rm{`knop4Sq)(}O_@gk{14y51)iOcGQ5J=b#e z2Yx^6^*F^F7q_m-AGFFgx5uqyw6_4w?yKCJKDGGprWyekr;X(!4CnM5_5?KgN=3qCm03 z##6k%kIU5%g!cCL(+aK>`Wd;dZ4h$h_jb7n?nqx5&o9cUJfr%h#m4+Bh)>HodKcDcsXDXwzJ3jR(sSFqWV(OKHC*cV8;;&bH=ZI0YbW3PgIHwTjiWy z?2MXWO2u0RAEEq(zv9e%Rsz|0(OKB?_3*kkXwHxEuazIZ7=JhaNV*P~hv57q55LoebmJpfHXA@yuS{Esg+ z*C}0V-`x^=0nOa@SPUJek>td~tJ{U1T&m)~`FLp*4DF77S^{|0g%|JIqd-=5)p6a` zpJOsEkKT(FPS@t^80V!I-YJbLE@{5KmVXjEq{QbCnir%}3 zB)-J379=wrBNK6rbUL7Mh^tVmQYn-BJJP=n?P&m-7)P#OZjQoK0{5?}XqJScV6>QX zPR>G{xvU_P;q!;S9Y7*07=Z!=wxIUorMQP(m?te~6&Z0PXQ@I=EYhD*XomZ^z;`Os z4>Uh4)Cg2_##mUa>i1Dxi+R~g#!!i{?SMj%9rfaBPlWj_Yk)lCV--e^&3INB>I?lu z9YXCY5(9U`3o?w2Xa5ErMbl5+pDVpu8v+KJzI9{KFk1H?(1`_W>Cu903Hg81vEX32l{nP2vROa1Fi!Wou0+ZX7Rp`g;B$*Ni3MC-vZ`f zFTi7}c+D)!4hz6NH2e%%t_;tkA0nfkmhLtRW%){TpIqD_ev>}#mVc)<$-1GKO_oK8 zy$CF^aV#x7>F4-J;P@tqWKG0|D1+7h+{ZHU5OVjh>#aa8+V;6BQ)8L5k9t`>)>7zr zfIlv77^`Fvm<)_+^z@ac%D&hnlUAFt8!x=jdaUo{)M9Ar;Tz5Dcd_|~Hl6CaRnK3R zYn${wZe8_BZ0l0c%qbP}>($jsNDay>8+JG@F!uV4F;#zGsBP0f$f3HqEHDz_sCr^q z1;1}7KJ9&`AX2Qdav1(nNzz+GPdEk5K3;hGXe{Hq13{)c zZy%fFEEH#nlJoG{f*M^#8yXuW%!9svN8ry-Vi7AOFnN~r&D`%6d#lvMXBgZkX^vFj z;tkent^62jUr$Cc^@y31Lka6hS>F?1tE8JW$iXO*n9CQMk}D*At3U(-W1E~z>tG?> z5f`5R5LbrhRNR8kv&5d9SL7ke2a*Xr)Qp#75 z6?-p035n2<7hK;sb>t9GAwG4{9v~iEIG>}7B5zcCgZhu$M0-z8?eUO^E?g)md^XT_ z2^~-u$yak>LBy(=*GsTj6p<>b5PO&un@5hGCxpBQlOB3DpsItKZRC*oXq-r{u}Wb; z&ko>#fbnl2Z;o@KqS-d6DTeCG?m1 z&E>p}SEc*)SD&QjZbs!Csjx~0+$@ekuzV_wAalnQvX3a^n~3ui)|rDO+9HW|JPEeBGP4 z)?zcZ<8qv47`EWA*_X~H^vr(lP|f%=%cWFM;u)OFHruKT<~?>5Y8l?56>&;=WdZU# zZEK4-C8s-3zPMA^&y~e*9z)!ZJghr3N^pJa2A$??Xqx-BR*TytGYor&l8Q+^^r%Yq02xay^f#;;wO6K7G!v>wRd6531WnDI~h$PN( z+4#08uX?r&zVKsQ;?5eBX=FxsXaGyH4Gth4a&L|{8LnNCHFr1M{KjJ!BfBS_aiy-E zxtmNcXq3}WTwQ7Dq-9YS5o758sT(5b`Sg-NcH>M9OH1oW6&sZ@|GYk|cJI`vm zO<$~q!3_$&GfWetudRc*mp8)M)q7DEY-#@8w=ItkApfq3sa)*GRqofuL7)dafznKf zLuembr#8gm*lIqKH)KMxSDqbik*B(1bFt%3Vv|ypehXLCa&wc7#u!cJNlUfWs8iQ` z$66(F=1fkxwg745-8_eqV>nWGY3DjB9gE23$R5g&w|C{|xvT@7j*@aZNB199scGchI7pINb5iyqYn)O=yJJX)Ca3&Ca+{n<=1w|(|f0)h<9gs$pVSV<<9Og-V z8ki@nKwE)x)^wmHBMk?mpMT=g{S#^8W|>&rI#Ceh;9za}io0k@0JxiCqi-jHlxbt3 zjJA?RihhRvhk6%G5-D{ePh1jare*fQS<328P-DcVAxPTrw=n6k?C6EV75f}cnBRPT zMYDqqKu(ND&aOtc!QRV`vzJSVxx8i~WB#5Ml{b#eQqNnSi7l-bS-`ITW<^zyYQA(b zbj4SuRK>q9o`_v%+C=S?h>2e4!66Ij(P5{7Uz$3u6YJJC$W%EoBa{-(=tQ|y1vov%ZkXVOV z##_UVg4V^4ne#4~<-1DkJqkKqgT+E_=&4Ue&eQ-JC+gi?7G@d6= zximz{zE)WW{b@QCJ!7l&N5x=dXS?$5RBU-VvN4Uec-GHK&jPa&P2z+qDdLhIB+HU) zu0CW&uLvE^4I5xtK-$+oe|58)7m6*PO%Xt<+-XEA%jG_BEachkF3e@pn?tl!`8lOF zbi2QOuNXX)YT*MCYflILO{VZ*9GiC%R4FO20zMK?p+&aCMm2oeMK7(aW=UDzr=AO0 z$5mJ%=qRsR8rZ>_YsL+vi{3*J_9Kzq(;ZwRj+4_f0-*wbkSMPWahX#Fj_a8BnrhJ6 zo^ZZ?Vah1@&6#r=JkuaYDBdp;J3@ii+CHM&@9*er&#P}$@wI$bfrH)&c!*|nkvhf%^*Y6b%dKz%QBSIo@U z{?V^qEs4`q<8@n+u8YiB^sc@6g>TncG<|GsmC3egwE6aO=EwLr~3-2 zNr`+)`i+-83?|1Xy0^8ps&pb}YT?w1eWVnC9Ps1=KM;Rw)bH6O!7Did1NwpnqVPZc z*%Qo~qkDL>@^<^fmIBtx$WUWQiNtAB2x-LO^BB=|w~-zTnJNEdm1Ou(?8PF&U88X@ z#8rdaTd||)dG^uJw~N_-%!XNbuAyh4`>Shea=pSj0TqP+w4!`nxsmVSv02kb`DBr% zyX=e>5IJ3JYPtdbCHvKMdhXUO_*E9jc_?se7%VJF#&ZaBD;7+eFN3x+hER7!u&`Wz z7zMvBPR4y`*$a250KYjFhAKS%*XG&c;R-kS0wNY1=836wL6q02mqx;IPcH(6ThA@2 zXKQF|9H>6AW$KUF#^A%l6y5{fel77_+cR_zZ0(7=6bmNXABv}R!B-{(E^O6Y?ZS)n zs1QEmh_Fm7p}oRyT3zxUNr4UV8NGs+2b8|4shO$OGFj3D&7_e?#yDi=TTe%$2QbG5 zk<;q7aQ;p!M-Osm{vFdmXZ@!z9uWh!;*%>(vTRggufuUGP9Hols@vhx z73pn$3u2;vzRvnXuT&$Os7J@6y12*j!{ix%3B4YU1466ItmJs0NsU(4ZYRYh7wEA6q{b*Hs6@k~ zi7Yq@Ax!et0cUMTvk7P%ym){MHpcliHEI~e3HP0NV=}7;xFv#IC?a<=`>~j_sk{e> z7vg-tK*p83HZ0=QK@ zRIHo^r{D8&Ms-^WZp+6US_Quqjh$Q66W^1}=Uz&XJ8AQE9&2}P zY|FXZzZ|0IiaBd2qdt6dIjQr(ZMIOU%NG1F&fu6Po9m^?BvLhI6T0R!H2d8;U(&p2 zYA|MFscMqcO(ye~Jp?F;0>Ke+5hzVr?aBNe>GsGgr$XrpS9uajN2kNQ3o$V5rp0T( z0$6TJC;3)26SNG#XcX7l^MKTn$ga?6r4Jzfb%ZgA(Zbwit0$kY=avSnI$@Gk%+^pu zS5mHrcRS8LFPC*uVWH4DDD1pY$H8N>X?KIJZuZ2SvTqc5Nr0GHdD8TCJcd$zIhOdC zZX0ErnsozQh;t^==4zTfrZO421AL?)O)l#GSxU#|LTTg4#&yeK=^w#;q63!Nv~1(@ zs^-RNRuF&qgcr+bIzc@7$h9L;_yjdifE*$j0Q&Np=1AuHL--zdkv@}`1 zo~LlDl_YAq*z?vmr4M`GjDkl9?p|-tl(DtX76oZv25_DtZutLS9Ez!5~p?th@4 zyc_uax4W#<(#)LMkvo)yp|5tKsC2=p#6PyhpH|449T<9Zdk|%CAb5cw?fhvQtBO&7 zpQ9$24yLqPHP;$N&fe2wm%8qdctwIna<3SwGtQA3{C77s%CW%LYxtK(SBGustL0<( zu~U9r0UOkr(c{OJxZS0Ntu3+cJlF7R`7k-Bsa&q?9Ae5{{|o~?cM+T7{lB1^#vT8R z?>c9fNWey`1dKDY%F3d2O*8^qYhjlB8*7HMKE<*=(A`{>=1%s1}Pm&#_t1xy!FkPk@%SMEka2@*= zxDuM|vJJ5s+xgDls{>*o!7eOcs|xuVBPWX&+y5vEiADK%hi`#Dbd>;;Pbk2H4*-X&R?_-6ZEutSd8hC+sSjhIo z;D(j4P;2EVpEj#UF7IjM6PC+X$C5T&=nL`*!*hm9U)#O?>wqOgC>jXKN3Slk_yaQX zLf|4D8T4k|wHW`;#ZQVocNF|3izi0sOqXzi7@KlYC3CXBG`94wD;tMI1bj|8Vm zY}9`VI9!plSfhAal$M_HlaYOVNU?9Z#0<$o?lXXbX3O(l_?f)i3_~r+GcO-x#+x^X zfsZl0>Rj2iP1rsT;+b;Mr? z4Vu&O)Q5ru4j;qaSP5gA{az@XTS1NpT0d9Xhl_FkkRpcEGA0(QQ~YMh#&zwDUkNzm z6cgkdgl9W{iL6ArJ1TQHqnQ^SQ1WGu?FT|93$Ba}mPCH~!$3}0Y0g zcoG%bdTd$bmBx9Y<`Jc+=Cp4}c@EUfjiz;Rcz101p z=?#i$wo>gBE9|szaZMt-d4nUIhBnYRuBVyx+p?5#aZQgUe(!ah`J#l1$%bl5avL27 zU2~@V`3Ic&!?FhDX@Cw!R4%xtWark#p8DLT)HCZ?VJxf^yr@AD*!ERK3#L$E^*Yr? zzN&uF9Roh4rP+r`Z#7U$tzl6>k!b~HgM$C<_crP=vC>6=q{j?(I}!9>g3rJU(&){o z`R^E*9%+kEa8H_fkD9VT7(Fks&Y-RcHaUJYf-|B+eMXMaRM;{FKRiTB>1(=Iij4k1(X__|WqAd-~t#2@UQ}Z&<1Th0azdXfoll!dd)6>1miA z!&=6sDJm=e$?L&06+Q3`D-HNSkK-3$3DdZMX-6Xjn;wd#9A{~ur!2NcX>(qY_oZL0~H7dnQ9sgLe!W>~2|RSW7|hWn<({Pg*xF$%B-!rKe^_R_vc z(LO!0agxxP;FWPV({8#lEv$&&GVakGus=@!3YVG`y^AO1m{2%Np;>HNA1e{=?ra1C}H zAwT0sbwG|!am;fl?*_t^^#yLDXZ*Nx)_FqueZi0c-G~omtpHW0Cu)mEJ`Z1X8brq$ z%vK##b~o*^b&Hz!hgrD=^6P8}aW40lhzMLB5T5*v`1QH?+L~-@CDi3+C@nRf2{7UE zyDIe{@LKw`Eu=Z%6<<_=#V|yxJIKiq_N?ZJ_v0$c)N4l07ZV_mIXG}glfBSPivOhw z-~+9GdckSpMBNR9eR`Y|9_)sXS+u_OiQ%!9rE(2AFjoxN8lk16Sb~^Sq6kRoEp3yD(mm`HsYIXcag_EAB8MHc}nahxVVUTts~U9P|f;7Ul$_` zStR4v&P4q_$KXOEni$lkxy8=9w8G&47VY0oDb^+jT+>ARe3NHUg~St`$RDxY)?;_F znqTujR&chZd2qHF7y8D$4&E3+e@J~!X3&BW4BF(Ebp#TEjrd+9SU!)j;qH+ZkL@AW z?J6Mj}v0_+D zH0qlbzCkHf|EZ`6c>5ig5NAFF%|La%M-}g(7&}Vx8K)qg30YD;H!S!??{;YivzrH0 z(M%2*b_S-)yh&Aiqai)GF^c!<1Xemj|13>dZ_M#)41SrP;OEMaRJ)bCeX*ZT7W`4Y zQ|8L@NHpD@Tf(5>1U(s5iW~Zdf7$@pAL`a3X@YUv1J>q-uJ_(Dy5nYTCUHC}1(dlI zt;5>DLcHh&jbysqt?G01MhXI3!8wgf){Hv}=0N|L$t8M#L7d6WscO8Om2|NBz2Ga^ zs86y%x$H18)~akOWD7@em7)ldlWgb?_sRN>-EcYQO_}aX@+b$dR{146>{kXWP4$nN{V0_+|3{Lt|8uX_fhKh~i{(x%cj*PU$i{PO(5$uA? zQzO>a6oPj-TUk&{zq?JD2MNb6Mf~V3g$ra+PB;ujLJ2JM(a7N*b`y{MX--!fAd}5C zF$D_b8S;+Np(!cW)(hnv5b@@|EMt*RLKF*wy>ykFhEhlPN~n_Bj>LT9B^_yj>z#fx z3JuE4H&?Cc!;G@}E*3k`HK#8ag`yE3Z1)5JUlSua%qkF zkTu|<9{w9OSi$qr)WD#7EzITnch=xnR63E*d~WGvi*Co9BBE?ETHud;!Z)7&wz+l6 zuKODYG1>I1U#a%&(GNJ`AqRfg=H!BtSl+_;CEeufF-#+*2EMMz-22@>18=8PH{PHd z);mN=aR0MPF>eutLiS#-AOX>#2%+pTGEOj!j4L(m0~&xR=0+g#HNpno6@veLhJp}e zyNVC$a>4;!9&iGvU_dj&xbKt@^t6r%f^)+}eV^suRTLP52+BVs0kOLwg6n`=NUv50E7My8XQUh?y%mW62OT1pMrKI3Q(r`7vU&@93=G~A?b(^pvC-8x=bSk zZ60BQR96WB1Z@9Df(M1IQh+YrU8sEjB=Tc2;(zBn-pete*icZE|M&Uc+oHg`|1o`g zH~m+k=D$o);{Rs)b<9Zo|9_Z6L6QHLNki(N>Dw^^i1LITprZeeqIaT#+)fw)PlllU zldphHC)t!0Gf(i9zgVm>`*TbmITF zH1FZ4{wrjRCx{t^26VK_2srZuWuY*EMAsMrJYFFCH35Ky7bq8<0K|ey2wHnrFMZyr z&^yEgX{{3i@&iE5>xKZ{Ads36G3a!i50D!C4?^~cLB<<|fc1!XN(HJRM)H^21sEs%vv+Mu0h*HkLHaEffMwc0n6)JhNXY#M5w@iO@dfXY z0c6dM2a4Hd1SA*#qYj@jK}uVgAZdaBj8t6uuhUNe>)ne9vfd#C6qLV9+@Q7{MnF#0 zJ7fd-ivG_~u3bVvOzpcw1u~ZSp8-kl(sunnX>L~*K-ByWDM2E8>;Si6kn^58AZQxI xVa^It*?521mj4+UJO?7%w*+`EfEcU=@KhDx-s^WzP+ae~{CgHDE&XryzW}Nww%-5% diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index aa991fceae..ae04661ee7 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 1b6c787337..a69d9cb6c2 100755 --- a/gradlew +++ b/gradlew @@ -205,6 +205,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index ac1b06f938..53a6b238d4 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -75,13 +75,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal From af32d322abb2439941e430b4fc074f3f4ea623a8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:06:49 -0700 Subject: [PATCH 11/25] fix(deps): update dependency io.openlineage:openlineage-java to v0.14.1 (#2109) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Willy Lulciuc --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 49996bd3bd..6fdceb0e49 100644 --- a/build.gradle +++ b/build.gradle @@ -60,7 +60,7 @@ subprojects { junit5Version = '5.8.2' lombokVersion = '1.18.24' mockitoVersion = '4.5.1' - openlineageVersion = '0.13.0' + openlineageVersion = '0.14.1' slf4jVersion = '1.7.36' postgresqlVersion = '42.3.7' isReleaseVersion = !version.endsWith('SNAPSHOT') From 0a52a5c9c11773941a3f2a8d611c8ec91f4198f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:07:38 -0700 Subject: [PATCH 12/25] Bump terser from 5.12.0 to 5.15.0 in /web (#2123) Bumps [terser](https://github.com/terser/terser) from 5.12.0 to 5.15.0. - [Release notes](https://github.com/terser/terser/releases) - [Changelog](https://github.com/terser/terser/blob/master/CHANGELOG.md) - [Commits](https://github.com/terser/terser/compare/v5.12.0...v5.15.0) --- updated-dependencies: - dependency-name: terser dependency-type: indirect ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Willy Lulciuc --- web/package-lock.json | 126 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 103 insertions(+), 23 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index 88af3e021e..19f924498c 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1603,6 +1603,58 @@ "node": ">=8" } }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", + "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.15", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.15.tgz", + "integrity": "sha512-oWZNOULl+UbhsgB51uuZzglikfIKSUBO/M9W2OfEjn7cmqoAiCgmv9lyACTUacZwBz0ITnJ2NqjU8Tx0DHL88g==", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@material-ui/core": { "version": "4.12.3", "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.12.3.tgz", @@ -14907,13 +14959,13 @@ } }, "node_modules/terser": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.12.0.tgz", - "integrity": "sha512-R3AUhNBGWiFc77HXag+1fXpAxTAFRQTJemlJKjAgD9r8xXTpjNKqIXwHM/o7Rh+O0kUJtS3WQVdBeMKFk5sw9A==", + "version": "5.15.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.0.tgz", + "integrity": "sha512-L1BJiXVmheAQQy+as0oF3Pwtlo4s3Wi1X2zNZ2NxOB4wx9bdS9Vk67XQENLFdLYGCK/Z2di53mTj/hBafR+dTA==", "dependencies": { + "@jridgewell/source-map": "^0.3.2", "acorn": "^8.5.0", "commander": "^2.20.0", - "source-map": "~0.7.2", "source-map-support": "~0.5.20" }, "bin": { @@ -14981,14 +15033,6 @@ "randombytes": "^2.1.0" } }, - "node_modules/terser/node_modules/source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", - "engines": { - "node": ">= 8" - } - }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -17671,6 +17715,49 @@ } } }, + "@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "requires": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==" + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==" + }, + "@jridgewell/source-map": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", + "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "requires": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" + }, + "@jridgewell/trace-mapping": { + "version": "0.3.15", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.15.tgz", + "integrity": "sha512-oWZNOULl+UbhsgB51uuZzglikfIKSUBO/M9W2OfEjn7cmqoAiCgmv9lyACTUacZwBz0ITnJ2NqjU8Tx0DHL88g==", + "requires": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "@material-ui/core": { "version": "4.12.3", "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.12.3.tgz", @@ -28158,21 +28245,14 @@ } }, "terser": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.12.0.tgz", - "integrity": "sha512-R3AUhNBGWiFc77HXag+1fXpAxTAFRQTJemlJKjAgD9r8xXTpjNKqIXwHM/o7Rh+O0kUJtS3WQVdBeMKFk5sw9A==", + "version": "5.15.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.0.tgz", + "integrity": "sha512-L1BJiXVmheAQQy+as0oF3Pwtlo4s3Wi1X2zNZ2NxOB4wx9bdS9Vk67XQENLFdLYGCK/Z2di53mTj/hBafR+dTA==", "requires": { + "@jridgewell/source-map": "^0.3.2", "acorn": "^8.5.0", "commander": "^2.20.0", - "source-map": "~0.7.2", "source-map-support": "~0.5.20" - }, - "dependencies": { - "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" - } } }, "terser-webpack-plugin": { From e60da31c985fb8a87ee7d307f8f9d3d3f949c9a2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:08:04 -0700 Subject: [PATCH 13/25] fix(deps): update dependency com.diffplug.spotless:spotless-plugin-gradle to v6.11.0 (#2107) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6fdceb0e49..58d28085e6 100644 --- a/build.gradle +++ b/build.gradle @@ -21,7 +21,7 @@ buildscript { dependencies { classpath 'com.adarshr:gradle-test-logger-plugin:3.2.0' classpath 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2' - classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.7.2' + classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.11.0' } } From 59dd1efe63e6382ec5d2329ac36981e9b6d8be65 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:08:56 -0700 Subject: [PATCH 14/25] fix(deps): update dependency org.assertj:assertj-core to v3.23.1 (#2111) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 58d28085e6..8144e6a17e 100644 --- a/build.gradle +++ b/build.gradle @@ -54,7 +54,7 @@ subprojects { } ext { - assertjVersion = '3.22.0' + assertjVersion = '3.23.1' dropwizardVersion = '2.1.1' jacocoVersion = '0.8.8' junit5Version = '5.8.2' From 9b7da052b7aca74a01aca017c2e098878ae52195 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:14:51 -0700 Subject: [PATCH 15/25] chore(deps): update helm/chart-testing-action action to v2.3.0 (#2105) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/test-chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-chart.yaml b/.github/workflows/test-chart.yaml index fc52ed1886..836e57c472 100644 --- a/.github/workflows/test-chart.yaml +++ b/.github/workflows/test-chart.yaml @@ -24,7 +24,7 @@ jobs: python-version: 3.7 - name: Setup chart-testing - uses: helm/chart-testing-action@v2.2.1 + uses: helm/chart-testing-action@v2.3.0 - name: Run chart-testing (list-changed) id: list-changed From a3f50b21fea0f55b2018f21ce760741257c46726 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:15:41 -0700 Subject: [PATCH 16/25] chore(deps): update helm/kind-action action to v1.4.0 (#2106) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Willy Lulciuc --- .github/workflows/test-chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-chart.yaml b/.github/workflows/test-chart.yaml index 836e57c472..8d08b70c69 100644 --- a/.github/workflows/test-chart.yaml +++ b/.github/workflows/test-chart.yaml @@ -38,7 +38,7 @@ jobs: run: ct lint --config ct.yaml - name: Create kind cluster - uses: helm/kind-action@v1.2.0 + uses: helm/kind-action@v1.4.0 - name: Run chart-testing (install) run: ct install --config ct.yaml From 0022008f47e39148c55c4235d15a47caa9b4f0b9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:16:17 -0700 Subject: [PATCH 17/25] fix(deps): update dependency com.graphql-java:graphql-java to v18.3 (#2108) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Willy Lulciuc --- api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/build.gradle b/api/build.gradle index 5615b0aa1a..0b0654fb32 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -48,7 +48,7 @@ dependencies { implementation "io.sentry:sentry:${sentryVersion}" implementation 'org.flywaydb:flyway-core:8.5.13' implementation "org.postgresql:postgresql:${postgresqlVersion}" - implementation 'com.graphql-java:graphql-java:18.1' + implementation 'com.graphql-java:graphql-java:18.3' implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0' testImplementation "io.dropwizard:dropwizard-testing:${dropwizardVersion}" From 5431dabe0765a14894a16e3b5b4671f39b5116f4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Sep 2022 14:32:17 -0700 Subject: [PATCH 18/25] fix(deps): update dependency org.apache.maven:maven-archiver to v3.6.0 (#2110) * fix(deps): update dependency org.apache.maven:maven-archiver to v3.6.0 Signed-off-by: Renovate Bot * Add commons-lang3 Signed-off-by: wslulciuc Signed-off-by: Renovate Bot Signed-off-by: wslulciuc Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: wslulciuc --- clients/java/build.gradle | 3 ++- clients/java/src/main/java/marquez/client/MarquezPathV1.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clients/java/build.gradle b/clients/java/build.gradle index c056245f41..7fea543a13 100644 --- a/clients/java/build.gradle +++ b/clients/java/build.gradle @@ -22,8 +22,9 @@ plugins { dependencies { implementation "io.dropwizard:dropwizard-jackson:${dropwizardVersion}" implementation "org.slf4j:slf4j-api:${slf4jVersion}" + implementation 'org.apache.commons:commons-lang3:3.12.0' implementation 'org.apache.httpcomponents:httpclient:4.5.13' - implementation 'org.apache.maven:maven-archiver:3.5.2' + implementation 'org.apache.maven:maven-archiver:3.6.0' testImplementation "org.slf4j:slf4j-simple:${slf4jVersion}" } diff --git a/clients/java/src/main/java/marquez/client/MarquezPathV1.java b/clients/java/src/main/java/marquez/client/MarquezPathV1.java index 2c679ac548..11f2eaf0e3 100644 --- a/clients/java/src/main/java/marquez/client/MarquezPathV1.java +++ b/clients/java/src/main/java/marquez/client/MarquezPathV1.java @@ -16,7 +16,7 @@ import javax.annotation.Nullable; import lombok.NonNull; import marquez.client.models.RunState; -import org.apache.maven.shared.utils.StringUtils; +import org.apache.commons.lang3.StringUtils; class MarquezPathV1 { @VisibleForTesting static final String BASE_PATH = "/api/v1"; From 87e26533dd904a0504420dca8a79dc9c3cddf3ca Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:37:45 -0700 Subject: [PATCH 19/25] fix(deps): update mockito monorepo to v4.8.0 (#2140) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8144e6a17e..a1211cc826 100644 --- a/build.gradle +++ b/build.gradle @@ -59,7 +59,7 @@ subprojects { jacocoVersion = '0.8.8' junit5Version = '5.8.2' lombokVersion = '1.18.24' - mockitoVersion = '4.5.1' + mockitoVersion = '4.8.0' openlineageVersion = '0.14.1' slf4jVersion = '1.7.36' postgresqlVersion = '42.3.7' From cdbabcf4d302a6f57a62409f00d4a5db9328395f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:38:30 -0700 Subject: [PATCH 20/25] fix(deps): update dropwizard monorepo to v2.1.2 (#2136) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a1211cc826..fad5b29ca7 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,7 @@ subprojects { ext { assertjVersion = '3.23.1' - dropwizardVersion = '2.1.1' + dropwizardVersion = '2.1.2' jacocoVersion = '0.8.8' junit5Version = '5.8.2' lombokVersion = '1.18.24' From 5e47e0e0260d82f975535441b8a9baba0dad73cd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:38:47 -0700 Subject: [PATCH 21/25] fix(deps): update junit5 monorepo to v5.9.1 (#2139) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index fad5b29ca7..ecc8657966 100644 --- a/build.gradle +++ b/build.gradle @@ -57,7 +57,7 @@ subprojects { assertjVersion = '3.23.1' dropwizardVersion = '2.1.2' jacocoVersion = '0.8.8' - junit5Version = '5.8.2' + junit5Version = '5.9.1' lombokVersion = '1.18.24' mockitoVersion = '4.8.0' openlineageVersion = '0.14.1' From 0a3e3e7fde6c466b450449b18748c5b1f5da76e7 Mon Sep 17 00:00:00 2001 From: Willy Lulciuc Date: Tue, 27 Sep 2022 10:00:27 -0700 Subject: [PATCH 22/25] Use `clean` when running `shadowJar` in Dockerfile (#2145) * Add clean to build cmd for docker Signed-off-by: wslulciuc * Remove MARQUEZ_VERSION usage in docker/entrypoint.sh Signed-off-by: wslulciuc Signed-off-by: wslulciuc --- .dockerignore | 2 +- Dockerfile | 2 +- docker/entrypoint.sh | 8 +------- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index a3b1498b04..22225457ca 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ *.md .* Dockerfile -build +api/build docs web/node_modules diff --git a/Dockerfile b/Dockerfile index 27374adae5..a8fdf1ccdc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ COPY build.gradle build.gradle COPY api ./api COPY api/build.gradle ./api/build.gradle COPY clients/java ./clients/java -RUN ./gradlew --no-daemon :api:shadowJar +RUN ./gradlew --no-daemon clean :api:shadowJar FROM eclipse-temurin:17 RUN apt-get update && apt-get install -y postgresql-client bash coreutils diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 3e279e97cf..5320c72cc9 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -12,14 +12,8 @@ if [[ -z "${MARQUEZ_CONFIG}" ]]; then echo "WARNING 'MARQUEZ_CONFIG' not set, using development configuration." fi -if [[ -z "${MARQUEZ_VERSION}" ]]; then - MARQUEZ_VERSION='*' - echo "WARNING 'MARQUEZ_VERSION' not set. Running could fail if directory contains multiple jar versions." -fi - - # Adjust java options for the http server JAVA_OPTS="${JAVA_OPTS} -Duser.timezone=UTC -Dlog4j2.formatMsgNoLookups=true" # Start http server with java options and configuration -java ${JAVA_OPTS} -jar marquez-${MARQUEZ_VERSION}.jar server ${MARQUEZ_CONFIG} +java ${JAVA_OPTS} -jar marquez-*.jar server ${MARQUEZ_CONFIG} From c5fc6bf073a1a3f568c01173f62f931a3c1a4d4f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 27 Sep 2022 10:02:39 -0700 Subject: [PATCH 23/25] fix(deps): update dependency org.postgresql:postgresql to v42.5.0 (#2138) Signed-off-by: Renovate Bot Signed-off-by: Renovate Bot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ecc8657966..9476b6cd20 100644 --- a/build.gradle +++ b/build.gradle @@ -62,7 +62,7 @@ subprojects { mockitoVersion = '4.8.0' openlineageVersion = '0.14.1' slf4jVersion = '1.7.36' - postgresqlVersion = '42.3.7' + postgresqlVersion = '42.5.0' isReleaseVersion = !version.endsWith('SNAPSHOT') } From bb3d163146473b135367bd875e71aaf7fd2f9ace Mon Sep 17 00:00:00 2001 From: Michael Collado <40346148+collado-mike@users.noreply.github.com> Date: Tue, 27 Sep 2022 11:50:58 -0700 Subject: [PATCH 24/25] Update insert job function to avoid joining on symlinks for jobs that have no symlinks (#2144) Signed-off-by: Michael Collado Signed-off-by: Michael Collado --- .../R__1_Jobs_view_and_rewrite_function.sql | 54 +++++++++++++++---- .../test/java/marquez/db/LineageDaoTest.java | 5 +- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql b/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql index a3a2b488ae..a544c0352e 100644 --- a/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql +++ b/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql @@ -21,13 +21,14 @@ SELECT f.uuid, FROM jobs_fqn f, jobs j WHERE j.uuid = f.uuid -AND j.is_hidden IS FALSE; + AND j.is_hidden IS FALSE; CREATE OR REPLACE FUNCTION rewrite_jobs_fqn_table() RETURNS TRIGGER AS $$ DECLARE job_uuid uuid; + job_updated_at timestamp with time zone; new_symlink_target_uuid uuid; old_symlink_target_uuid uuid; inserted_job jobs_view%rowtype; @@ -53,7 +54,7 @@ BEGIN COALESCE(NEW.parent_job_uuid::char(36), ''), false ON CONFLICT (name, namespace_uuid, parent_job_uuid_string) - DO UPDATE SET updated_at = EXCLUDED.updated_at, + DO UPDATE SET updated_at = now(), type = EXCLUDED.type, description = EXCLUDED.description, current_job_context_uuid = EXCLUDED.current_job_context_uuid, @@ -64,15 +65,48 @@ BEGIN EXCLUDED.symlink_target_uuid), is_hidden = false -- the SELECT statement below will get the OLD symlink_target_uuid in case of update and the NEW - -- version in case of insert - RETURNING uuid, symlink_target_uuid, (SELECT symlink_target_uuid FROM jobs j2 WHERE j2.uuid=jobs.uuid) - INTO job_uuid, new_symlink_target_uuid, old_symlink_target_uuid; + -- version in case of insert + RETURNING uuid, + updated_at, + symlink_target_uuid, + (SELECT symlink_target_uuid FROM jobs j2 WHERE j2.uuid=jobs.uuid) + INTO job_uuid, job_updated_at, new_symlink_target_uuid, old_symlink_target_uuid; - -- update the jobs_fqn table only when inserting a new record (NEW.uuid will equal the job_uuid - -- when inserting a new record) or when the symlink_target_uuid is being updated. - IF NEW.uuid = job_uuid OR - (new_symlink_target_uuid IS DISTINCT FROM old_symlink_target_uuid) THEN - RAISE LOG 'Updating jobs_fqn due to % to job % (%)', TG_OP, NEW.name, job_uuid; + + -- update the jobs_fqn table when inserting a new record + -- (NEW.uuid will equal the job_uuid when inserting a new record) + -- AND if the symlink target is null + -- Avoid constructing the symlinks and aliases, as that is expensive + IF TG_OP='INSERT' + AND NEW.uuid = job_uuid + AND NEW.symlink_target_uuid IS NULL + AND NEW.updated_at=job_updated_at THEN + RAISE DEBUG 'Inserting into jobs_fqn for new job % (%)', NEW.name, job_uuid; + WITH fqn AS (SELECT j.uuid, + CASE + WHEN j.parent_job_uuid IS NULL THEN j.name + ELSE jf.job_fqn || '.' || j.name + END AS name, + j.namespace_uuid, + j.namespace_name, + jf.job_fqn AS parent_job_name, + j.parent_job_uuid + FROM jobs j + LEFT JOIN jobs_fqn jf ON jf.uuid=j.parent_job_uuid + WHERE j.uuid=job_uuid) + INSERT + INTO jobs_fqn + SELECT j.uuid, + jf.namespace_uuid, + jf.namespace_name, + jf.parent_job_name, + ARRAY[jf.name]::text[], + jf.name AS job_fqn + FROM jobs j + INNER JOIN fqn jf ON jf.uuid = j.uuid; + -- or when the symlink_target_uuid is being updated. + ELSIF (new_symlink_target_uuid IS NOT NULL AND new_symlink_target_uuid IS DISTINCT FROM old_symlink_target_uuid) THEN + RAISE DEBUG 'Updating jobs_fqn due to % to job % (%)', TG_OP, NEW.name, job_uuid; WITH RECURSIVE jobs_symlink AS (SELECT j.uuid, j.uuid AS link_target_uuid, j.symlink_target_uuid FROM jobs j diff --git a/api/src/test/java/marquez/db/LineageDaoTest.java b/api/src/test/java/marquez/db/LineageDaoTest.java index 07b2b9ee7c..f0a451bbda 100644 --- a/api/src/test/java/marquez/db/LineageDaoTest.java +++ b/api/src/test/java/marquez/db/LineageDaoTest.java @@ -15,6 +15,7 @@ import com.google.common.base.Functions; import java.sql.SQLException; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -220,7 +221,7 @@ public void testGetLineageForSymlinkedJob() throws SQLException { .upsertJob( UUID.randomUUID(), JobType.valueOf(writeJob.getJob().getType()), - writeJob.getJob().getCreatedAt(), + Instant.now(), namespaceRow.getUuid(), writeJob.getJob().getNamespaceName(), symlinkTargetJobName, @@ -233,7 +234,7 @@ public void testGetLineageForSymlinkedJob() throws SQLException { .upsertJob( writeJob.getJob().getUuid(), JobType.valueOf(writeJob.getJob().getType()), - writeJob.getJob().getCreatedAt(), + Instant.now(), namespaceRow.getUuid(), writeJob.getJob().getNamespaceName(), writeJob.getJob().getName(), From 2909864e62d209444c69fb48a92dd31a70f02e1b Mon Sep 17 00:00:00 2001 From: "pawel.leszczynski" Date: Wed, 28 Sep 2022 09:05:04 +0200 Subject: [PATCH 25/25] dataset symlinks provided (#2087) Signed-off-by: Pawel Leszczynski Signed-off-by: Pawel Leszczynski --- CHANGELOG.md | 2 + api/src/main/java/marquez/db/BaseDao.java | 3 + api/src/main/java/marquez/db/Columns.java | 3 + api/src/main/java/marquez/db/DatasetDao.java | 26 +++++---- .../java/marquez/db/DatasetSymlinkDao.java | 51 +++++++++++++++++ .../main/java/marquez/db/OpenLineageDao.java | 38 ++++++++++++- .../db/mappers/DatasetSymlinksRowMapper.java | 36 ++++++++++++ .../marquez/db/models/DatasetSymlinkRow.java | 33 +++++++++++ .../marquez/service/models/LineageEvent.java | 38 ++++++++++++- .../db/migration/R__3_Datasets_view.sql | 36 ++++++------ .../db/migration/V48__dataset_symlinks.sql | 24 ++++++++ .../test/java/marquez/db/DatasetDaoTest.java | 28 ++++++++++ .../test/java/marquez/db/LineageDaoTest.java | 1 + .../java/marquez/db/OpenLineageDaoTest.java | 56 +++++++++++++++++++ api/src/test/java/marquez/db/RunDaoTest.java | 1 + .../marquez/service/LineageServiceTest.java | 1 + 16 files changed, 348 insertions(+), 29 deletions(-) create mode 100644 api/src/main/java/marquez/db/DatasetSymlinkDao.java create mode 100644 api/src/main/java/marquez/db/mappers/DatasetSymlinksRowMapper.java create mode 100644 api/src/main/java/marquez/db/models/DatasetSymlinkRow.java create mode 100644 api/src/main/resources/marquez/db/migration/V48__dataset_symlinks.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 69e267abb1..11d8cc1057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.26.0...HEAD) ### Fixed * Add support for `parentRun` facet as reported by older Airflow OpenLineage versions [@collado-mike](https://github.com/collado-mike) +* Implemented dataset symlink feature which allows providing multiple names for a dataset and adds edges to lineage graph based on symlinks [`#2066`](https://github.com/MarquezProject/marquez/pull/2066) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski) + ## [0.26.0](https://github.com/MarquezProject/marquez/compare/0.25.0...0.26.0) - 2022-09-15 ### Added diff --git a/api/src/main/java/marquez/db/BaseDao.java b/api/src/main/java/marquez/db/BaseDao.java index 848372d863..f21a7c67f3 100644 --- a/api/src/main/java/marquez/db/BaseDao.java +++ b/api/src/main/java/marquez/db/BaseDao.java @@ -30,6 +30,9 @@ public interface BaseDao extends SqlObject { @CreateSqlObject NamespaceDao createNamespaceDao(); + @CreateSqlObject + DatasetSymlinkDao createDatasetSymlinkDao(); + @CreateSqlObject RunDao createRunDao(); diff --git a/api/src/main/java/marquez/db/Columns.java b/api/src/main/java/marquez/db/Columns.java index 0cfe4d1e3a..f1642c1295 100644 --- a/api/src/main/java/marquez/db/Columns.java +++ b/api/src/main/java/marquez/db/Columns.java @@ -80,6 +80,9 @@ private Columns() {} public static final String FIELD_UUIDS = "field_uuids"; public static final String LIFECYCLE_STATE = "lifecycle_state"; + /* DATASET SYMLINK ROW COLUMNS */ + public static final String IS_PRIMARY = "is_primary"; + /* STREAM VERSION ROW COLUMNS */ public static final String SCHEMA_LOCATION = "schema_location"; diff --git a/api/src/main/java/marquez/db/DatasetDao.java b/api/src/main/java/marquez/db/DatasetDao.java index 54f8cf93b5..64f7faee25 100644 --- a/api/src/main/java/marquez/db/DatasetDao.java +++ b/api/src/main/java/marquez/db/DatasetDao.java @@ -23,6 +23,7 @@ import marquez.db.mappers.DatasetMapper; import marquez.db.mappers.DatasetRowMapper; import marquez.db.models.DatasetRow; +import marquez.db.models.DatasetSymlinkRow; import marquez.db.models.DatasetVersionRow; import marquez.db.models.NamespaceRow; import marquez.db.models.SourceRow; @@ -73,8 +74,7 @@ void updateLastModifiedAt( WITH selected_datasets AS ( SELECT d.* FROM datasets_view d - WHERE d.namespace_name = :namespaceName - AND d.name = :datasetName + WHERE CAST((:namespaceName, :datasetName) AS DATASET_NAME) = ANY(d.dataset_symlinks) ), dataset_runs AS ( SELECT d.uuid, d.name, d.namespace_name, dv.run_uuid, dv.lifecycle_state, event_time, event FROM selected_datasets d @@ -229,7 +229,7 @@ INSERT INTO datasets ( :description, :isDeleted, false - ) ON CONFLICT (namespace_uuid, name) + ) ON CONFLICT (uuid) DO UPDATE SET type = EXCLUDED.type, updated_at = EXCLUDED.updated_at, @@ -275,7 +275,7 @@ DatasetRow upsert( + ":sourceName, " + ":name, " + ":physicalName) " - + "ON CONFLICT (namespace_uuid, name) " + + "ON CONFLICT (uuid) " + "DO UPDATE SET " + "type = EXCLUDED.type, " + "updated_at = EXCLUDED.updated_at, " @@ -296,8 +296,10 @@ DatasetRow upsert( """ UPDATE datasets SET is_hidden = true - WHERE namespace_name = :namespaceName - AND name = :name + FROM dataset_symlinks, namespaces + WHERE dataset_symlinks.dataset_uuid = datasets.uuid + AND namespaces.uuid = dataset_symlinks.namespace_uuid + AND namespaces.name=:namespaceName AND dataset_symlinks.name=:name RETURNING * """) Optional delete(String namespaceName, String name); @@ -310,6 +312,10 @@ default Dataset upsertDatasetMeta( createNamespaceDao() .upsertNamespaceRow( UUID.randomUUID(), now, namespaceName.getValue(), DEFAULT_NAMESPACE_OWNER); + DatasetSymlinkRow symlinkRow = + createDatasetSymlinkDao() + .upsertDatasetSymlinkRow( + UUID.randomUUID(), datasetName.getValue(), namespaceRow.getUuid(), true, null, now); SourceRow sourceRow = createSourceDao() .upsertOrDefault( @@ -318,13 +324,12 @@ default Dataset upsertDatasetMeta( now, datasetMeta.getSourceName().getValue(), ""); - UUID newDatasetUuid = UUID.randomUUID(); DatasetRow datasetRow; if (datasetMeta.getDescription().isPresent()) { datasetRow = upsert( - newDatasetUuid, + symlinkRow.getUuid(), datasetMeta.getType(), now, namespaceRow.getUuid(), @@ -338,7 +343,7 @@ default Dataset upsertDatasetMeta( } else { datasetRow = upsert( - newDatasetUuid, + symlinkRow.getUuid(), datasetMeta.getType(), now, namespaceRow.getUuid(), @@ -349,7 +354,8 @@ default Dataset upsertDatasetMeta( datasetMeta.getPhysicalName().getValue()); } - updateDatasetMetric(namespaceName, datasetMeta.getType(), newDatasetUuid, datasetRow.getUuid()); + updateDatasetMetric( + namespaceName, datasetMeta.getType(), symlinkRow.getUuid(), datasetRow.getUuid()); TagDao tagDao = createTagDao(); List datasetTagMappings = new ArrayList<>(); diff --git a/api/src/main/java/marquez/db/DatasetSymlinkDao.java b/api/src/main/java/marquez/db/DatasetSymlinkDao.java new file mode 100644 index 0000000000..a79ac9ed34 --- /dev/null +++ b/api/src/main/java/marquez/db/DatasetSymlinkDao.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018-2022 contributors to the Marquez project + * SPDX-License-Identifier: Apache-2.0 + */ + +package marquez.db; + +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import marquez.db.mappers.DatasetSymlinksRowMapper; +import marquez.db.models.DatasetSymlinkRow; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; + +@RegisterRowMapper(DatasetSymlinksRowMapper.class) +public interface DatasetSymlinkDao extends BaseDao { + + default DatasetSymlinkRow upsertDatasetSymlinkRow( + UUID uuid, String name, UUID namespaceUuid, boolean isPrimary, String type, Instant now) { + doUpsertDatasetSymlinkRow(uuid, name, namespaceUuid, isPrimary, type, now); + return findDatasetSymlinkByNamespaceUuidAndName(namespaceUuid, name).orElseThrow(); + } + + @SqlQuery("SELECT * FROM dataset_symlinks WHERE namespace_uuid = :namespaceUuid and name = :name") + Optional findDatasetSymlinkByNamespaceUuidAndName( + UUID namespaceUuid, String name); + + @SqlUpdate( + """ + INSERT INTO dataset_symlinks ( + dataset_uuid, + name, + namespace_uuid, + is_primary, + type, + created_at, + updated_at + ) VALUES ( + :uuid, + :name, + :namespaceUuid, + :isPrimary, + :type, + :now, + :now) + ON CONFLICT (name, namespace_uuid) DO NOTHING""") + void doUpsertDatasetSymlinkRow( + UUID uuid, String name, UUID namespaceUuid, boolean isPrimary, String type, Instant now); +} diff --git a/api/src/main/java/marquez/db/OpenLineageDao.java b/api/src/main/java/marquez/db/OpenLineageDao.java index 75ce7f4b70..c05fe71146 100644 --- a/api/src/main/java/marquez/db/OpenLineageDao.java +++ b/api/src/main/java/marquez/db/OpenLineageDao.java @@ -33,6 +33,7 @@ import marquez.db.mappers.LineageEventMapper; import marquez.db.models.DatasetFieldRow; import marquez.db.models.DatasetRow; +import marquez.db.models.DatasetSymlinkRow; import marquez.db.models.DatasetVersionRow; import marquez.db.models.JobContextRow; import marquez.db.models.JobRow; @@ -120,6 +121,7 @@ default UpdateLineageRow updateMarquezModel(LineageEvent event, ObjectMapper map default UpdateLineageRow updateBaseMarquezModel(LineageEvent event, ObjectMapper mapper) { NamespaceDao namespaceDao = createNamespaceDao(); + DatasetSymlinkDao datasetSymlinkDao = createDatasetSymlinkDao(); DatasetDao datasetDao = createDatasetDao(); SourceDao sourceDao = createSourceDao(); JobDao jobDao = createJobDao(); @@ -316,6 +318,7 @@ default UpdateLineageRow updateBaseMarquezModel(LineageEvent event, ObjectMapper runUuid, true, namespaceDao, + datasetSymlinkDao, sourceDao, datasetDao, datasetVersionDao, @@ -337,6 +340,7 @@ default UpdateLineageRow updateBaseMarquezModel(LineageEvent event, ObjectMapper runUuid, false, namespaceDao, + datasetSymlinkDao, sourceDao, datasetDao, datasetVersionDao, @@ -532,6 +536,7 @@ default DatasetRecord upsertLineageDataset( UUID runUuid, boolean isInput, NamespaceDao namespaceDao, + DatasetSymlinkDao datasetSymlinkDao, SourceDao sourceDao, DatasetDao datasetDao, DatasetVersionDao datasetVersionDao, @@ -568,6 +573,35 @@ default DatasetRecord upsertLineageDataset( formatNamespaceName(ds.getNamespace()), DEFAULT_NAMESPACE_OWNER); + DatasetSymlinkRow symlink = + datasetSymlinkDao.upsertDatasetSymlinkRow( + UUID.randomUUID(), + formatDatasetName(ds.getName()), + dsNamespace.getUuid(), + true, + null, + now); + + Optional.ofNullable(ds.getFacets()) + .map(facets -> facets.getSymlinks()) + .ifPresent( + el -> + el.getIdentifiers().stream() + .forEach( + id -> + datasetSymlinkDao.doUpsertDatasetSymlinkRow( + symlink.getUuid(), + id.getName(), + namespaceDao + .upsertNamespaceRow( + UUID.randomUUID(), + now, + id.getNamespace(), + DEFAULT_NAMESPACE_OWNER) + .getUuid(), + false, + id.getType(), + now))); String dslifecycleState = Optional.ofNullable(ds.getFacets()) .map(DatasetFacets::getLifecycleStateChange) @@ -576,7 +610,7 @@ default DatasetRecord upsertLineageDataset( DatasetRow datasetRow = datasetDao.upsert( - UUID.randomUUID(), + symlink.getUuid(), getDatasetType(ds), now, datasetNamespace.getUuid(), @@ -609,7 +643,7 @@ default DatasetRecord upsertLineageDataset( dsNamespace.getName(), source.getName(), dsRow.getPhysicalName(), - dsRow.getName(), + symlink.getName(), dslifecycleState, fields, runUuid) diff --git a/api/src/main/java/marquez/db/mappers/DatasetSymlinksRowMapper.java b/api/src/main/java/marquez/db/mappers/DatasetSymlinksRowMapper.java new file mode 100644 index 0000000000..ff70f0be6c --- /dev/null +++ b/api/src/main/java/marquez/db/mappers/DatasetSymlinksRowMapper.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018-2022 contributors to the Marquez project + * SPDX-License-Identifier: Apache-2.0 + */ + +package marquez.db.mappers; + +import static marquez.db.Columns.booleanOrDefault; +import static marquez.db.Columns.stringOrNull; +import static marquez.db.Columns.stringOrThrow; +import static marquez.db.Columns.timestampOrThrow; +import static marquez.db.Columns.uuidOrThrow; + +import java.sql.ResultSet; +import java.sql.SQLException; +import lombok.NonNull; +import marquez.db.Columns; +import marquez.db.models.DatasetSymlinkRow; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; + +public class DatasetSymlinksRowMapper implements RowMapper { + + @Override + public DatasetSymlinkRow map(@NonNull ResultSet results, @NonNull StatementContext context) + throws SQLException { + return new DatasetSymlinkRow( + uuidOrThrow(results, Columns.DATASET_UUID), + stringOrThrow(results, Columns.NAME), + uuidOrThrow(results, Columns.NAMESPACE_UUID), + stringOrNull(results, Columns.TYPE), + booleanOrDefault(results, Columns.IS_PRIMARY, false), + timestampOrThrow(results, Columns.CREATED_AT), + timestampOrThrow(results, Columns.UPDATED_AT)); + } +} diff --git a/api/src/main/java/marquez/db/models/DatasetSymlinkRow.java b/api/src/main/java/marquez/db/models/DatasetSymlinkRow.java new file mode 100644 index 0000000000..de10b91109 --- /dev/null +++ b/api/src/main/java/marquez/db/models/DatasetSymlinkRow.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018-2022 contributors to the Marquez project + * SPDX-License-Identifier: Apache-2.0 + */ + +package marquez.db.models; + +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import javax.annotation.Nullable; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NonNull; +import lombok.Value; + +@AllArgsConstructor +@EqualsAndHashCode +@Value +public class DatasetSymlinkRow { + @NonNull UUID uuid; + @NonNull String name; + @NonNull UUID namespaceUuid; + @Nullable String type; + @NonNull boolean isPrimary; + @Getter @NonNull private final Instant createdAt; + @Getter @NonNull private final Instant updatedAt; + + public Optional getType() { + return Optional.ofNullable(type); + } +} diff --git a/api/src/main/java/marquez/service/models/LineageEvent.java b/api/src/main/java/marquez/service/models/LineageEvent.java index 1de50c2b7b..5065ad7ffa 100644 --- a/api/src/main/java/marquez/service/models/LineageEvent.java +++ b/api/src/main/java/marquez/service/models/LineageEvent.java @@ -322,7 +322,8 @@ public static class Dataset extends BaseJsonModel { "schema", "dataSource", "description", - "lifecycleStateChange" + "lifecycleStateChange", + "symlinks" }) public static class DatasetFacets { @@ -330,6 +331,7 @@ public static class DatasetFacets { @Valid private SchemaDatasetFacet schema; @Valid private LifecycleStateChangeFacet lifecycleStateChange; @Valid private DatasourceDatasetFacet dataSource; + @Valid private DatasetSymlinkFacet symlinks; private String description; @Builder.Default @JsonIgnore private Map additional = new LinkedHashMap<>(); @@ -351,6 +353,10 @@ public SchemaDatasetFacet getSchema() { return schema; } + public DatasetSymlinkFacet getSymlinks() { + return symlinks; + } + public LifecycleStateChangeFacet getLifecycleStateChange() { return lifecycleStateChange; } @@ -412,6 +418,36 @@ public static class SchemaField extends BaseJsonModel { private String description; } + @NoArgsConstructor + @Getter + @Setter + @Valid + @ToString + public static class DatasetSymlinkFacet extends BaseFacet { + + @Valid private List identifiers; + + @Builder + public DatasetSymlinkFacet( + @NotNull URI _producer, @NotNull URI _schemaURL, List identifiers) { + super(_producer, _schemaURL); + this.identifiers = identifiers; + } + } + + @AllArgsConstructor + @NoArgsConstructor + @Setter + @Getter + @Valid + @ToString + public static class SymlinkIdentifier extends BaseJsonModel { + + @NotNull private String namespace; + @NotNull private String name; + @Nullable private String type; + } + @NoArgsConstructor @Getter @Setter diff --git a/api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql b/api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql index d33bd843b3..20e90dcd5f 100644 --- a/api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql +++ b/api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql @@ -1,18 +1,22 @@ CREATE OR REPLACE VIEW datasets_view AS -SELECT uuid, - type, - created_at, - updated_at, - namespace_uuid, - source_uuid, - name, - physical_name, - description, - current_version_uuid, - last_modified_at, - namespace_name, - source_name, - is_deleted -FROM datasets -WHERE is_hidden IS FALSE; \ No newline at end of file +SELECT d.uuid, + d.type, + d.created_at, + d.updated_at, + d.namespace_uuid, + d.source_uuid, + d.name, + array_agg(CAST((namespaces.name, symlinks.name) AS DATASET_NAME)) AS dataset_symlinks, + d.physical_name, + d.description, + d.current_version_uuid, + d.last_modified_at, + d.namespace_name, + d.source_name, + d.is_deleted +FROM datasets d +JOIN dataset_symlinks symlinks ON d.uuid = symlinks.dataset_uuid +INNER JOIN namespaces ON symlinks.namespace_uuid = namespaces.uuid +WHERE d.is_hidden IS FALSE +GROUP BY d.uuid; \ No newline at end of file diff --git a/api/src/main/resources/marquez/db/migration/V48__dataset_symlinks.sql b/api/src/main/resources/marquez/db/migration/V48__dataset_symlinks.sql new file mode 100644 index 0000000000..4ea94721fb --- /dev/null +++ b/api/src/main/resources/marquez/db/migration/V48__dataset_symlinks.sql @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +CREATE TABLE dataset_symlinks ( + dataset_uuid UUID, + name VARCHAR(255) NOT NULL, + namespace_uuid UUID REFERENCES namespaces(uuid), + type VARCHAR(64), + is_primary BOOLEAN DEFAULT FALSE, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + UNIQUE (namespace_uuid, name) +); + +CREATE INDEX dataset_symlinks_dataset_uuid on dataset_symlinks (dataset_uuid); + +INSERT INTO dataset_symlinks (dataset_uuid, name, namespace_uuid, is_primary, created_at, updated_at) +SELECT d.uuid, d.name, d.namespace_uuid, TRUE, d.created_at, d.updated_at FROM datasets d; + +DROP TYPE IF EXISTS DATASET_NAME; +CREATE TYPE DATASET_NAME AS ( + namespace VARCHAR(255), + name VARCHAR(255) +); + diff --git a/api/src/test/java/marquez/db/DatasetDaoTest.java b/api/src/test/java/marquez/db/DatasetDaoTest.java index c1ef2f7bfa..09e61d03d2 100644 --- a/api/src/test/java/marquez/db/DatasetDaoTest.java +++ b/api/src/test/java/marquez/db/DatasetDaoTest.java @@ -70,6 +70,7 @@ public void tearDown(Jdbi jdbi) { handle.execute("DELETE FROM dataset_versions_field_mapping"); handle.execute("DELETE FROM stream_versions"); handle.execute("DELETE FROM dataset_versions"); + handle.execute("DELETE FROM dataset_symlinks"); handle.execute("UPDATE runs SET start_run_state_uuid=NULL, end_run_state_uuid=NULL"); handle.execute("DELETE FROM run_states"); handle.execute("DELETE FROM runs"); @@ -195,6 +196,33 @@ public void testGetDatasetWithDatasetMarkedDeleted() { assertThat(datasetDao.findWithTags(NAMESPACE, DATASET).get().isDeleted()).isTrue(); } + @Test + public void testGetDatasetBySymlink() { + createLineageRow( + openLineageDao, + "aJob", + "COMPLETE", + jobFacet, + Collections.emptyList(), + Collections.singletonList( + new Dataset( + NAMESPACE, + DATASET, + LineageEvent.DatasetFacets.builder() + .symlinks( + new LineageEvent.DatasetSymlinkFacet( + PRODUCER_URL, + SCHEMA_URL, + Collections.singletonList( + new LineageEvent.SymlinkIdentifier( + "symlinkNamespace", "symlinkName", "type")))) + .build()))); + + // verify dataset is returned by its name and symlink name + assertThat(datasetDao.findDatasetByName(NAMESPACE, DATASET)).isPresent(); + assertThat(datasetDao.findDatasetByName("symlinkNamespace", "symlinkName")).isPresent(); + } + @Test public void testGetDatasetWithMultipleVersions() { createLineageRow( diff --git a/api/src/test/java/marquez/db/LineageDaoTest.java b/api/src/test/java/marquez/db/LineageDaoTest.java index f0a451bbda..0a0becbb9d 100644 --- a/api/src/test/java/marquez/db/LineageDaoTest.java +++ b/api/src/test/java/marquez/db/LineageDaoTest.java @@ -85,6 +85,7 @@ public void tearDown(Jdbi jdbi) { handle.execute("DELETE FROM runs_input_mapping"); handle.execute("DELETE FROM dataset_versions_field_mapping"); handle.execute("DELETE FROM dataset_versions"); + handle.execute("DELETE FROM dataset_symlinks"); handle.execute("UPDATE runs SET start_run_state_uuid=NULL, end_run_state_uuid=NULL"); handle.execute("DELETE FROM run_states"); handle.execute("DELETE FROM runs"); diff --git a/api/src/test/java/marquez/db/OpenLineageDaoTest.java b/api/src/test/java/marquez/db/OpenLineageDaoTest.java index dd5d7188cd..46c4254f1e 100644 --- a/api/src/test/java/marquez/db/OpenLineageDaoTest.java +++ b/api/src/test/java/marquez/db/OpenLineageDaoTest.java @@ -10,6 +10,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.util.Arrays; +import java.util.Collections; import java.util.List; import marquez.db.models.UpdateLineageRow; import marquez.db.models.UpdateLineageRow.DatasetRecord; @@ -34,6 +35,8 @@ class OpenLineageDaoTest { public static final String DATASET_NAME = "theDataset"; private static OpenLineageDao dao; + private static DatasetSymlinkDao symlinkDao; + private static NamespaceDao namespaceDao; private final DatasetFacets datasetFacets = LineageTestUtils.newDatasetFacet( new SchemaField("name", "STRING", "my name"), new SchemaField("age", "INT", "my age")); @@ -41,6 +44,8 @@ class OpenLineageDaoTest { @BeforeAll public static void setUpOnce(Jdbi jdbi) { dao = jdbi.onDemand(OpenLineageDao.class); + symlinkDao = jdbi.onDemand(DatasetSymlinkDao.class); + namespaceDao = jdbi.onDemand(NamespaceDao.class); } /** When reading a dataset, the version is assumed to be the version last written */ @@ -94,6 +99,56 @@ void testUpdateMarquezModelLifecycleStateChangeFacet() { .isEqualTo("TRUNCATE"); } + @Test + void testUpdateMarquezModelDatasetWithSymlinks() { + Dataset dataset = + new Dataset( + LineageTestUtils.NAMESPACE, + DATASET_NAME, + LineageEvent.DatasetFacets.builder() + .symlinks( + new LineageEvent.DatasetSymlinkFacet( + PRODUCER_URL, + SCHEMA_URL, + Collections.singletonList( + new LineageEvent.SymlinkIdentifier( + "symlinkNamespace", "symlinkName", "some-type")))) + .build()); + + JobFacet jobFacet = new JobFacet(null, null, null, LineageTestUtils.EMPTY_MAP); + UpdateLineageRow writeJob = + LineageTestUtils.createLineageRow( + dao, WRITE_JOB_NAME, "COMPLETE", jobFacet, Arrays.asList(), Arrays.asList(dataset)); + + UpdateLineageRow readJob = + LineageTestUtils.createLineageRow( + dao, + WRITE_JOB_NAME, + "COMPLETE", + jobFacet, + Arrays.asList( + new Dataset( + "symlinkNamespace", + "symlinkName", + LineageEvent.DatasetFacets.builder().build())), + Arrays.asList()); + + // make sure writeJob output dataset and readJob input dataset are the same (have the same uuid) + assertThat(writeJob.getOutputs()).isPresent().get().asList().size().isEqualTo(1); + assertThat(writeJob.getOutputs().get().get(0).getDatasetRow().getUuid()) + .isEqualTo(readJob.getInputs().get().get(0).getDatasetRow().getUuid()); + // make sure symlink is stored with type in dataset_symlinks table + assertThat( + symlinkDao + .findDatasetSymlinkByNamespaceUuidAndName( + namespaceDao.findNamespaceByName("symlinkNamespace").get().getUuid(), + "symlinkName") + .get() + .getType() + .get()) + .isEqualTo("some-type"); + } + /** * When reading a new dataset, a version is created and the dataset's current version is updated * immediately. @@ -146,6 +201,7 @@ void testUpdateMarquezModelWithNonMatchingReadSchema() { new SchemaField("eyeColor", "STRING", "my eye color"))), this.datasetFacets.getLifecycleStateChange(), this.datasetFacets.getDataSource(), + null, this.datasetFacets.getDescription(), this.datasetFacets.getAdditionalFacets()); UpdateLineageRow readJob = diff --git a/api/src/test/java/marquez/db/RunDaoTest.java b/api/src/test/java/marquez/db/RunDaoTest.java index 87b0d79925..8207f3ad5e 100644 --- a/api/src/test/java/marquez/db/RunDaoTest.java +++ b/api/src/test/java/marquez/db/RunDaoTest.java @@ -69,6 +69,7 @@ public void tearDown(Jdbi jdbi) { handle.execute("DELETE FROM datasets_tag_mapping"); handle.execute("DELETE FROM dataset_versions_field_mapping"); handle.execute("DELETE FROM dataset_versions"); + handle.execute("DELETE FROM dataset_symlinks"); handle.execute("UPDATE runs SET start_run_state_uuid=NULL, end_run_state_uuid=NULL"); handle.execute("DELETE FROM run_states"); handle.execute("DELETE FROM runs"); diff --git a/api/src/test/java/marquez/service/LineageServiceTest.java b/api/src/test/java/marquez/service/LineageServiceTest.java index 86c6a4252c..2879640d85 100644 --- a/api/src/test/java/marquez/service/LineageServiceTest.java +++ b/api/src/test/java/marquez/service/LineageServiceTest.java @@ -88,6 +88,7 @@ public void tearDown(Jdbi jdbi) { handle.execute("DELETE FROM dataset_versions_field_mapping"); handle.execute("DELETE FROM stream_versions"); handle.execute("DELETE FROM dataset_versions"); + handle.execute("DELETE FROM dataset_symlinks"); handle.execute("UPDATE runs SET start_run_state_uuid=NULL, end_run_state_uuid=NULL"); handle.execute("DELETE FROM run_states"); handle.execute("DELETE FROM runs");