diff --git a/.circleci/config.yml b/.circleci/config.yml index ea43b23a18..8fcb3e11d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,10 @@ only-on-release: &only-on-release branches: ignore: /.*/ +orbs: + # https://circleci.com/orbs/registry/orb/codecov/codecov + codecov: codecov/codecov@3.2.3 + jobs: build-api: working_directory: ~/marquez @@ -32,7 +36,7 @@ jobs: - run: ./gradlew --no-daemon --stacktrace api:javadoc - run: ./gradlew --no-daemon --stacktrace api:build - run: ./gradlew --no-daemon api:jacocoTestReport - - run: bash <(curl -s https://codecov.io/bash) + - codecov/upload - store_test_results: path: api/build/test-results/test - store_artifacts: @@ -99,7 +103,7 @@ jobs: - run: ./gradlew --no-daemon --stacktrace clients:java:javadoc - run: ./gradlew --no-daemon --stacktrace clients:java:build - run: ./gradlew --no-daemon clients:java:jacocoTestReport - - run: bash <(curl -s https://codecov.io/bash) + - codecov/upload - store_test_results: path: clients/java/build/test-results/test - store_artifacts: @@ -119,7 +123,7 @@ jobs: - run: pip install -e .[dev] - run: flake8 - run: pytest --cov=marquez_python tests/ - - run: bash <(curl -s https://codecov.io/bash) + - codecov/upload build-client-python: working_directory: ~/marquez/clients/python diff --git a/.dockerignore b/.dockerignore index a3b1498b04..22225457ca 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ *.md .* Dockerfile -build +api/build docs web/node_modules diff --git a/.env.example b/.env.example index 106ad5d23a..d89c8f268f 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ API_PORT=5000 API_ADMIN_PORT=5001 WEB_PORT=3000 -TAG=0.19.0 +TAG=0.26.0 diff --git a/.github/workflows/headerchecker.yml b/.github/workflows/headerchecker.yml new file mode 100644 index 0000000000..5c232a43cc --- /dev/null +++ b/.github/workflows/headerchecker.yml @@ -0,0 +1,47 @@ +--- +name: Check for Headers in Source Code + +on: + push: + branches: ['renovate/**'] + pull_request: + branches: [main] + +jobs: + build: + name: Check for Headers + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - id: files + uses: jitterbit/get-changed-files@v1 + with: + format: 'json' + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Check for headers + run: | + ok=1 + readarray -t files <<<"$(jq -r '.[]' <<<'${{ steps.files.outputs.all }}')" + for file in ${files[@]}; do + if [[ ($file == *".java") ]]; then + if ! grep -q Copyright "$file"; then + ok=0 + echo "Copyright header not found in $file" + fi + fi + done + if [[ $ok == 0 ]]; then + exit 1 + else + GREEN="\e[32m" + echo -e "${GREEN}All changed & added files have been scanned. Result: no headers are missing.${ENDCOLOR}" + fi \ No newline at end of file diff --git a/.github/workflows/test-chart.yaml b/.github/workflows/test-chart.yaml index 36584d84ee..8d08b70c69 100644 --- a/.github/workflows/test-chart.yaml +++ b/.github/workflows/test-chart.yaml @@ -16,7 +16,7 @@ jobs: fetch-depth: 0 - name: Setup Helm - uses: azure/setup-helm@v2.1 + uses: azure/setup-helm@v2.2 - name: Setup Python uses: actions/setup-python@v3 @@ -24,7 +24,7 @@ jobs: python-version: 3.7 - name: Setup chart-testing - uses: helm/chart-testing-action@v2.2.1 + uses: helm/chart-testing-action@v2.3.0 - name: Run chart-testing (list-changed) id: list-changed @@ -38,7 +38,7 @@ jobs: run: ct lint --config ct.yaml - name: Create kind cluster - uses: helm/kind-action@v1.2.0 + uses: helm/kind-action@v1.4.0 - name: Run chart-testing (install) run: ct install --config ct.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 66c0a3b927..ecdec36bb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,13 @@ # Changelog ## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.26.0...HEAD) + +### Added +* Implemented dataset symlink feature which allows providing multiple names for a dataset and adds edges to lineage graph based on symlinks [`#2066`](https://github.com/MarquezProject/marquez/pull/2066) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski) * Store column lineage facets in separate table [`#2096`](https://github.com/MarquezProject/marquez/pull/2096) [@mzareba382](https://github.com/mzareba382) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski) +### Fixed +* Add support for `parentRun` facet as reported by older Airflow OpenLineage versions [@collado-mike](https://github.com/collado-mike) ## [0.26.0](https://github.com/MarquezProject/marquez/compare/0.25.0...0.26.0) - 2022-09-15 diff --git a/Dockerfile b/Dockerfile index 27374adae5..a8fdf1ccdc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ COPY build.gradle build.gradle COPY api ./api COPY api/build.gradle ./api/build.gradle COPY clients/java ./clients/java -RUN ./gradlew --no-daemon :api:shadowJar +RUN ./gradlew --no-daemon clean :api:shadowJar FROM eclipse-temurin:17 RUN apt-get update && apt-get install -y postgresql-client bash coreutils diff --git a/api/build.gradle b/api/build.gradle index 5615b0aa1a..0b0654fb32 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -48,7 +48,7 @@ dependencies { implementation "io.sentry:sentry:${sentryVersion}" implementation 'org.flywaydb:flyway-core:8.5.13' implementation "org.postgresql:postgresql:${postgresqlVersion}" - implementation 'com.graphql-java:graphql-java:18.1' + implementation 'com.graphql-java:graphql-java:18.3' implementation 'com.graphql-java-kickstart:graphql-java-servlet:12.0.0' testImplementation "io.dropwizard:dropwizard-testing:${dropwizardVersion}" diff --git a/api/src/main/java/marquez/db/BaseDao.java b/api/src/main/java/marquez/db/BaseDao.java index ac5e2919e7..5fc537e1d5 100644 --- a/api/src/main/java/marquez/db/BaseDao.java +++ b/api/src/main/java/marquez/db/BaseDao.java @@ -30,6 +30,9 @@ public interface BaseDao extends SqlObject { @CreateSqlObject NamespaceDao createNamespaceDao(); + @CreateSqlObject + DatasetSymlinkDao createDatasetSymlinkDao(); + @CreateSqlObject RunDao createRunDao(); diff --git a/api/src/main/java/marquez/db/Columns.java b/api/src/main/java/marquez/db/Columns.java index d53af0a09d..b01f9f33d0 100644 --- a/api/src/main/java/marquez/db/Columns.java +++ b/api/src/main/java/marquez/db/Columns.java @@ -80,6 +80,9 @@ private Columns() {} public static final String FIELD_UUIDS = "field_uuids"; public static final String LIFECYCLE_STATE = "lifecycle_state"; + /* DATASET SYMLINK ROW COLUMNS */ + public static final String IS_PRIMARY = "is_primary"; + /* STREAM VERSION ROW COLUMNS */ public static final String SCHEMA_LOCATION = "schema_location"; diff --git a/api/src/main/java/marquez/db/DatasetDao.java b/api/src/main/java/marquez/db/DatasetDao.java index 54f8cf93b5..64f7faee25 100644 --- a/api/src/main/java/marquez/db/DatasetDao.java +++ b/api/src/main/java/marquez/db/DatasetDao.java @@ -23,6 +23,7 @@ import marquez.db.mappers.DatasetMapper; import marquez.db.mappers.DatasetRowMapper; import marquez.db.models.DatasetRow; +import marquez.db.models.DatasetSymlinkRow; import marquez.db.models.DatasetVersionRow; import marquez.db.models.NamespaceRow; import marquez.db.models.SourceRow; @@ -73,8 +74,7 @@ void updateLastModifiedAt( WITH selected_datasets AS ( SELECT d.* FROM datasets_view d - WHERE d.namespace_name = :namespaceName - AND d.name = :datasetName + WHERE CAST((:namespaceName, :datasetName) AS DATASET_NAME) = ANY(d.dataset_symlinks) ), dataset_runs AS ( SELECT d.uuid, d.name, d.namespace_name, dv.run_uuid, dv.lifecycle_state, event_time, event FROM selected_datasets d @@ -229,7 +229,7 @@ INSERT INTO datasets ( :description, :isDeleted, false - ) ON CONFLICT (namespace_uuid, name) + ) ON CONFLICT (uuid) DO UPDATE SET type = EXCLUDED.type, updated_at = EXCLUDED.updated_at, @@ -275,7 +275,7 @@ DatasetRow upsert( + ":sourceName, " + ":name, " + ":physicalName) " - + "ON CONFLICT (namespace_uuid, name) " + + "ON CONFLICT (uuid) " + "DO UPDATE SET " + "type = EXCLUDED.type, " + "updated_at = EXCLUDED.updated_at, " @@ -296,8 +296,10 @@ DatasetRow upsert( """ UPDATE datasets SET is_hidden = true - WHERE namespace_name = :namespaceName - AND name = :name + FROM dataset_symlinks, namespaces + WHERE dataset_symlinks.dataset_uuid = datasets.uuid + AND namespaces.uuid = dataset_symlinks.namespace_uuid + AND namespaces.name=:namespaceName AND dataset_symlinks.name=:name RETURNING * """) Optional delete(String namespaceName, String name); @@ -310,6 +312,10 @@ default Dataset upsertDatasetMeta( createNamespaceDao() .upsertNamespaceRow( UUID.randomUUID(), now, namespaceName.getValue(), DEFAULT_NAMESPACE_OWNER); + DatasetSymlinkRow symlinkRow = + createDatasetSymlinkDao() + .upsertDatasetSymlinkRow( + UUID.randomUUID(), datasetName.getValue(), namespaceRow.getUuid(), true, null, now); SourceRow sourceRow = createSourceDao() .upsertOrDefault( @@ -318,13 +324,12 @@ default Dataset upsertDatasetMeta( now, datasetMeta.getSourceName().getValue(), ""); - UUID newDatasetUuid = UUID.randomUUID(); DatasetRow datasetRow; if (datasetMeta.getDescription().isPresent()) { datasetRow = upsert( - newDatasetUuid, + symlinkRow.getUuid(), datasetMeta.getType(), now, namespaceRow.getUuid(), @@ -338,7 +343,7 @@ default Dataset upsertDatasetMeta( } else { datasetRow = upsert( - newDatasetUuid, + symlinkRow.getUuid(), datasetMeta.getType(), now, namespaceRow.getUuid(), @@ -349,7 +354,8 @@ default Dataset upsertDatasetMeta( datasetMeta.getPhysicalName().getValue()); } - updateDatasetMetric(namespaceName, datasetMeta.getType(), newDatasetUuid, datasetRow.getUuid()); + updateDatasetMetric( + namespaceName, datasetMeta.getType(), symlinkRow.getUuid(), datasetRow.getUuid()); TagDao tagDao = createTagDao(); List datasetTagMappings = new ArrayList<>(); diff --git a/api/src/main/java/marquez/db/DatasetSymlinkDao.java b/api/src/main/java/marquez/db/DatasetSymlinkDao.java new file mode 100644 index 0000000000..a79ac9ed34 --- /dev/null +++ b/api/src/main/java/marquez/db/DatasetSymlinkDao.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018-2022 contributors to the Marquez project + * SPDX-License-Identifier: Apache-2.0 + */ + +package marquez.db; + +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import marquez.db.mappers.DatasetSymlinksRowMapper; +import marquez.db.models.DatasetSymlinkRow; +import org.jdbi.v3.sqlobject.config.RegisterRowMapper; +import org.jdbi.v3.sqlobject.statement.SqlQuery; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; + +@RegisterRowMapper(DatasetSymlinksRowMapper.class) +public interface DatasetSymlinkDao extends BaseDao { + + default DatasetSymlinkRow upsertDatasetSymlinkRow( + UUID uuid, String name, UUID namespaceUuid, boolean isPrimary, String type, Instant now) { + doUpsertDatasetSymlinkRow(uuid, name, namespaceUuid, isPrimary, type, now); + return findDatasetSymlinkByNamespaceUuidAndName(namespaceUuid, name).orElseThrow(); + } + + @SqlQuery("SELECT * FROM dataset_symlinks WHERE namespace_uuid = :namespaceUuid and name = :name") + Optional findDatasetSymlinkByNamespaceUuidAndName( + UUID namespaceUuid, String name); + + @SqlUpdate( + """ + INSERT INTO dataset_symlinks ( + dataset_uuid, + name, + namespace_uuid, + is_primary, + type, + created_at, + updated_at + ) VALUES ( + :uuid, + :name, + :namespaceUuid, + :isPrimary, + :type, + :now, + :now) + ON CONFLICT (name, namespace_uuid) DO NOTHING""") + void doUpsertDatasetSymlinkRow( + UUID uuid, String name, UUID namespaceUuid, boolean isPrimary, String type, Instant now); +} diff --git a/api/src/main/java/marquez/db/OpenLineageDao.java b/api/src/main/java/marquez/db/OpenLineageDao.java index d8a3ced6da..a45c7605e6 100644 --- a/api/src/main/java/marquez/db/OpenLineageDao.java +++ b/api/src/main/java/marquez/db/OpenLineageDao.java @@ -37,6 +37,7 @@ import marquez.db.models.ColumnLevelLineageRow; import marquez.db.models.DatasetFieldRow; import marquez.db.models.DatasetRow; +import marquez.db.models.DatasetSymlinkRow; import marquez.db.models.DatasetVersionRow; import marquez.db.models.InputFieldData; import marquez.db.models.JobContextRow; @@ -70,8 +71,8 @@ @RegisterRowMapper(LineageEventMapper.class) public interface OpenLineageDao extends BaseDao { - public String DEFAULT_SOURCE_NAME = "default"; - public String DEFAULT_NAMESPACE_OWNER = "anonymous"; + String DEFAULT_SOURCE_NAME = "default"; + String DEFAULT_NAMESPACE_OWNER = "anonymous"; @SqlUpdate( "INSERT INTO lineage_events (" @@ -126,6 +127,7 @@ default UpdateLineageRow updateMarquezModel(LineageEvent event, ObjectMapper map default UpdateLineageRow updateBaseMarquezModel(LineageEvent event, ObjectMapper mapper) { NamespaceDao namespaceDao = createNamespaceDao(); + DatasetSymlinkDao datasetSymlinkDao = createDatasetSymlinkDao(); DatasetDao datasetDao = createDatasetDao(); SourceDao sourceDao = createSourceDao(); JobDao jobDao = createJobDao(); @@ -323,6 +325,7 @@ default UpdateLineageRow updateBaseMarquezModel(LineageEvent event, ObjectMapper runUuid, true, namespaceDao, + datasetSymlinkDao, sourceDao, datasetDao, datasetVersionDao, @@ -345,6 +348,7 @@ default UpdateLineageRow updateBaseMarquezModel(LineageEvent event, ObjectMapper runUuid, false, namespaceDao, + datasetSymlinkDao, sourceDao, datasetDao, datasetVersionDao, @@ -379,7 +383,10 @@ private JobRow findParentJobRow( .findJobRowByRunUuid(uuid) .map( j -> { - String parentJobName = Utils.parseParentJobName(facet.getJob().getName()); + String parentJobName = + facet.getJob().getName().equals(event.getJob().getName()) + ? Utils.parseParentJobName(facet.getJob().getName()) + : facet.getJob().getName(); if (j.getNamespaceName().equals(facet.getJob().getNamespace()) && j.getName().equals(parentJobName)) { return j; @@ -441,6 +448,10 @@ private JobRow createParentJobRunRecord( PGobject inputs) { Instant now = event.getEventTime().withZoneSameInstant(ZoneId.of("UTC")).toInstant(); Logger log = LoggerFactory.getLogger(OpenLineageDao.class); + String parentJobName = + facet.getJob().getName().equals(event.getJob().getName()) + ? Utils.parseParentJobName(facet.getJob().getName()) + : facet.getJob().getName(); JobRow newParentJobRow = createJobDao() .upsertJob( @@ -449,7 +460,7 @@ private JobRow createParentJobRunRecord( now, namespace.getUuid(), namespace.getName(), - Utils.parseParentJobName(facet.getJob().getName()), + parentJobName, null, jobContext.getUuid(), location, @@ -534,6 +545,7 @@ default DatasetRecord upsertLineageDataset( UUID runUuid, boolean isInput, NamespaceDao namespaceDao, + DatasetSymlinkDao datasetSymlinkDao, SourceDao sourceDao, DatasetDao datasetDao, DatasetVersionDao datasetVersionDao, @@ -571,6 +583,35 @@ default DatasetRecord upsertLineageDataset( formatNamespaceName(ds.getNamespace()), DEFAULT_NAMESPACE_OWNER); + DatasetSymlinkRow symlink = + datasetSymlinkDao.upsertDatasetSymlinkRow( + UUID.randomUUID(), + formatDatasetName(ds.getName()), + dsNamespace.getUuid(), + true, + null, + now); + + Optional.ofNullable(ds.getFacets()) + .map(facets -> facets.getSymlinks()) + .ifPresent( + el -> + el.getIdentifiers().stream() + .forEach( + id -> + datasetSymlinkDao.doUpsertDatasetSymlinkRow( + symlink.getUuid(), + id.getName(), + namespaceDao + .upsertNamespaceRow( + UUID.randomUUID(), + now, + id.getNamespace(), + DEFAULT_NAMESPACE_OWNER) + .getUuid(), + false, + id.getType(), + now))); String dslifecycleState = Optional.ofNullable(ds.getFacets()) .map(DatasetFacets::getLifecycleStateChange) @@ -579,7 +620,7 @@ default DatasetRecord upsertLineageDataset( DatasetRow datasetRow = datasetDao.upsert( - UUID.randomUUID(), + symlink.getUuid(), getDatasetType(ds), now, datasetNamespace.getUuid(), @@ -612,7 +653,7 @@ default DatasetRecord upsertLineageDataset( dsNamespace.getName(), source.getName(), dsRow.getPhysicalName(), - dsRow.getName(), + symlink.getName(), dslifecycleState, fields, runUuid) diff --git a/api/src/main/java/marquez/db/mappers/DatasetSymlinksRowMapper.java b/api/src/main/java/marquez/db/mappers/DatasetSymlinksRowMapper.java new file mode 100644 index 0000000000..ff70f0be6c --- /dev/null +++ b/api/src/main/java/marquez/db/mappers/DatasetSymlinksRowMapper.java @@ -0,0 +1,36 @@ +/* + * Copyright 2018-2022 contributors to the Marquez project + * SPDX-License-Identifier: Apache-2.0 + */ + +package marquez.db.mappers; + +import static marquez.db.Columns.booleanOrDefault; +import static marquez.db.Columns.stringOrNull; +import static marquez.db.Columns.stringOrThrow; +import static marquez.db.Columns.timestampOrThrow; +import static marquez.db.Columns.uuidOrThrow; + +import java.sql.ResultSet; +import java.sql.SQLException; +import lombok.NonNull; +import marquez.db.Columns; +import marquez.db.models.DatasetSymlinkRow; +import org.jdbi.v3.core.mapper.RowMapper; +import org.jdbi.v3.core.statement.StatementContext; + +public class DatasetSymlinksRowMapper implements RowMapper { + + @Override + public DatasetSymlinkRow map(@NonNull ResultSet results, @NonNull StatementContext context) + throws SQLException { + return new DatasetSymlinkRow( + uuidOrThrow(results, Columns.DATASET_UUID), + stringOrThrow(results, Columns.NAME), + uuidOrThrow(results, Columns.NAMESPACE_UUID), + stringOrNull(results, Columns.TYPE), + booleanOrDefault(results, Columns.IS_PRIMARY, false), + timestampOrThrow(results, Columns.CREATED_AT), + timestampOrThrow(results, Columns.UPDATED_AT)); + } +} diff --git a/api/src/main/java/marquez/db/models/DatasetSymlinkRow.java b/api/src/main/java/marquez/db/models/DatasetSymlinkRow.java new file mode 100644 index 0000000000..de10b91109 --- /dev/null +++ b/api/src/main/java/marquez/db/models/DatasetSymlinkRow.java @@ -0,0 +1,33 @@ +/* + * Copyright 2018-2022 contributors to the Marquez project + * SPDX-License-Identifier: Apache-2.0 + */ + +package marquez.db.models; + +import java.time.Instant; +import java.util.Optional; +import java.util.UUID; +import javax.annotation.Nullable; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NonNull; +import lombok.Value; + +@AllArgsConstructor +@EqualsAndHashCode +@Value +public class DatasetSymlinkRow { + @NonNull UUID uuid; + @NonNull String name; + @NonNull UUID namespaceUuid; + @Nullable String type; + @NonNull boolean isPrimary; + @Getter @NonNull private final Instant createdAt; + @Getter @NonNull private final Instant updatedAt; + + public Optional getType() { + return Optional.ofNullable(type); + } +} diff --git a/api/src/main/java/marquez/service/OpenLineageService.java b/api/src/main/java/marquez/service/OpenLineageService.java index b8ac4ec64f..9c7deb4e24 100644 --- a/api/src/main/java/marquez/service/OpenLineageService.java +++ b/api/src/main/java/marquez/service/OpenLineageService.java @@ -30,6 +30,7 @@ import marquez.common.models.JobVersionId; import marquez.common.models.NamespaceName; import marquez.common.models.RunId; +import marquez.common.models.RunState; import marquez.db.BaseDao; import marquez.db.DatasetDao; import marquez.db.DatasetVersionDao; @@ -37,11 +38,13 @@ import marquez.db.models.JobRow; import marquez.db.models.RunArgsRow; import marquez.db.models.RunRow; +import marquez.db.models.RunStateRow; import marquez.db.models.UpdateLineageRow; import marquez.service.RunTransitionListener.JobInputUpdate; import marquez.service.RunTransitionListener.JobOutputUpdate; import marquez.service.RunTransitionListener.RunInput; import marquez.service.RunTransitionListener.RunOutput; +import marquez.service.RunTransitionListener.RunTransition; import marquez.service.models.LineageEvent; import marquez.service.models.RunMeta; @@ -91,6 +94,7 @@ public CompletableFuture createAsync(LineageEvent event) { buildJobOutputUpdate(update).ifPresent(runService::notify); } buildJobInputUpdate(update).ifPresent(runService::notify); + buildRunTransition(update).ifPresent(runService::notify); } }); @@ -222,4 +226,15 @@ private DatasetVersionId buildDatasetVersionId(ExtendedDatasetVersionRow ds) { .name(DatasetName.of(ds.getDatasetName())) .build(); } + + private Optional buildRunTransition(UpdateLineageRow record) { + RunId runId = RunId.of(record.getRun().getUuid()); + RunStateRow runStateRow = record.getRunState(); + if (runStateRow == null) { + return Optional.empty(); + } + RunState newState = RunState.valueOf(runStateRow.getState()); + RunState oldState = newState.isStarting() ? null : RunState.RUNNING; + return Optional.of(new RunTransition(runId, oldState, newState)); + } } diff --git a/api/src/main/java/marquez/service/models/LineageEvent.java b/api/src/main/java/marquez/service/models/LineageEvent.java index d4baeae8db..8202028944 100644 --- a/api/src/main/java/marquez/service/models/LineageEvent.java +++ b/api/src/main/java/marquez/service/models/LineageEvent.java @@ -5,6 +5,7 @@ package marquez.service.models; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -71,7 +72,11 @@ public static class Run extends BaseJsonModel { public static class RunFacet { @Valid private NominalTimeRunFacet nominalTime; - @Valid private ParentRunFacet parent; + + @JsonAlias( + "parentRun") // the Airflow integration previously reported parentRun instead of parent + @Valid + private ParentRunFacet parent; @Builder.Default @JsonIgnore private Map additional = new LinkedHashMap<>(); @@ -319,6 +324,7 @@ public static class Dataset extends BaseJsonModel { "description", "lifecycleStateChange", "columnLineage", + "symlinks" }) public static class DatasetFacets { @@ -327,6 +333,7 @@ public static class DatasetFacets { @Valid private LifecycleStateChangeFacet lifecycleStateChange; @Valid private DatasourceDatasetFacet dataSource; @Valid private ColumnLineageFacet columnLineage; + @Valid private DatasetSymlinkFacet symlinks; private String description; @Builder.Default @JsonIgnore private Map additional = new LinkedHashMap<>(); @@ -348,6 +355,10 @@ public SchemaDatasetFacet getSchema() { return schema; } + public DatasetSymlinkFacet getSymlinks() { + return symlinks; + } + public LifecycleStateChangeFacet getLifecycleStateChange() { return lifecycleStateChange; } @@ -413,6 +424,36 @@ public static class SchemaField extends BaseJsonModel { private String description; } + @NoArgsConstructor + @Getter + @Setter + @Valid + @ToString + public static class DatasetSymlinkFacet extends BaseFacet { + + @Valid private List identifiers; + + @Builder + public DatasetSymlinkFacet( + @NotNull URI _producer, @NotNull URI _schemaURL, List identifiers) { + super(_producer, _schemaURL); + this.identifiers = identifiers; + } + } + + @AllArgsConstructor + @NoArgsConstructor + @Setter + @Getter + @Valid + @ToString + public static class SymlinkIdentifier extends BaseJsonModel { + + @NotNull private String namespace; + @NotNull private String name; + @Nullable private String type; + } + @NoArgsConstructor @Getter @Setter diff --git a/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql b/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql index a3a2b488ae..a544c0352e 100644 --- a/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql +++ b/api/src/main/resources/marquez/db/migration/R__1_Jobs_view_and_rewrite_function.sql @@ -21,13 +21,14 @@ SELECT f.uuid, FROM jobs_fqn f, jobs j WHERE j.uuid = f.uuid -AND j.is_hidden IS FALSE; + AND j.is_hidden IS FALSE; CREATE OR REPLACE FUNCTION rewrite_jobs_fqn_table() RETURNS TRIGGER AS $$ DECLARE job_uuid uuid; + job_updated_at timestamp with time zone; new_symlink_target_uuid uuid; old_symlink_target_uuid uuid; inserted_job jobs_view%rowtype; @@ -53,7 +54,7 @@ BEGIN COALESCE(NEW.parent_job_uuid::char(36), ''), false ON CONFLICT (name, namespace_uuid, parent_job_uuid_string) - DO UPDATE SET updated_at = EXCLUDED.updated_at, + DO UPDATE SET updated_at = now(), type = EXCLUDED.type, description = EXCLUDED.description, current_job_context_uuid = EXCLUDED.current_job_context_uuid, @@ -64,15 +65,48 @@ BEGIN EXCLUDED.symlink_target_uuid), is_hidden = false -- the SELECT statement below will get the OLD symlink_target_uuid in case of update and the NEW - -- version in case of insert - RETURNING uuid, symlink_target_uuid, (SELECT symlink_target_uuid FROM jobs j2 WHERE j2.uuid=jobs.uuid) - INTO job_uuid, new_symlink_target_uuid, old_symlink_target_uuid; + -- version in case of insert + RETURNING uuid, + updated_at, + symlink_target_uuid, + (SELECT symlink_target_uuid FROM jobs j2 WHERE j2.uuid=jobs.uuid) + INTO job_uuid, job_updated_at, new_symlink_target_uuid, old_symlink_target_uuid; - -- update the jobs_fqn table only when inserting a new record (NEW.uuid will equal the job_uuid - -- when inserting a new record) or when the symlink_target_uuid is being updated. - IF NEW.uuid = job_uuid OR - (new_symlink_target_uuid IS DISTINCT FROM old_symlink_target_uuid) THEN - RAISE LOG 'Updating jobs_fqn due to % to job % (%)', TG_OP, NEW.name, job_uuid; + + -- update the jobs_fqn table when inserting a new record + -- (NEW.uuid will equal the job_uuid when inserting a new record) + -- AND if the symlink target is null + -- Avoid constructing the symlinks and aliases, as that is expensive + IF TG_OP='INSERT' + AND NEW.uuid = job_uuid + AND NEW.symlink_target_uuid IS NULL + AND NEW.updated_at=job_updated_at THEN + RAISE DEBUG 'Inserting into jobs_fqn for new job % (%)', NEW.name, job_uuid; + WITH fqn AS (SELECT j.uuid, + CASE + WHEN j.parent_job_uuid IS NULL THEN j.name + ELSE jf.job_fqn || '.' || j.name + END AS name, + j.namespace_uuid, + j.namespace_name, + jf.job_fqn AS parent_job_name, + j.parent_job_uuid + FROM jobs j + LEFT JOIN jobs_fqn jf ON jf.uuid=j.parent_job_uuid + WHERE j.uuid=job_uuid) + INSERT + INTO jobs_fqn + SELECT j.uuid, + jf.namespace_uuid, + jf.namespace_name, + jf.parent_job_name, + ARRAY[jf.name]::text[], + jf.name AS job_fqn + FROM jobs j + INNER JOIN fqn jf ON jf.uuid = j.uuid; + -- or when the symlink_target_uuid is being updated. + ELSIF (new_symlink_target_uuid IS NOT NULL AND new_symlink_target_uuid IS DISTINCT FROM old_symlink_target_uuid) THEN + RAISE DEBUG 'Updating jobs_fqn due to % to job % (%)', TG_OP, NEW.name, job_uuid; WITH RECURSIVE jobs_symlink AS (SELECT j.uuid, j.uuid AS link_target_uuid, j.symlink_target_uuid FROM jobs j diff --git a/api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql b/api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql index d33bd843b3..20e90dcd5f 100644 --- a/api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql +++ b/api/src/main/resources/marquez/db/migration/R__3_Datasets_view.sql @@ -1,18 +1,22 @@ CREATE OR REPLACE VIEW datasets_view AS -SELECT uuid, - type, - created_at, - updated_at, - namespace_uuid, - source_uuid, - name, - physical_name, - description, - current_version_uuid, - last_modified_at, - namespace_name, - source_name, - is_deleted -FROM datasets -WHERE is_hidden IS FALSE; \ No newline at end of file +SELECT d.uuid, + d.type, + d.created_at, + d.updated_at, + d.namespace_uuid, + d.source_uuid, + d.name, + array_agg(CAST((namespaces.name, symlinks.name) AS DATASET_NAME)) AS dataset_symlinks, + d.physical_name, + d.description, + d.current_version_uuid, + d.last_modified_at, + d.namespace_name, + d.source_name, + d.is_deleted +FROM datasets d +JOIN dataset_symlinks symlinks ON d.uuid = symlinks.dataset_uuid +INNER JOIN namespaces ON symlinks.namespace_uuid = namespaces.uuid +WHERE d.is_hidden IS FALSE +GROUP BY d.uuid; \ No newline at end of file diff --git a/api/src/main/resources/marquez/db/migration/V48__dataset_symlinks.sql b/api/src/main/resources/marquez/db/migration/V48__dataset_symlinks.sql new file mode 100644 index 0000000000..4ea94721fb --- /dev/null +++ b/api/src/main/resources/marquez/db/migration/V48__dataset_symlinks.sql @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: Apache-2.0 */ + +CREATE TABLE dataset_symlinks ( + dataset_uuid UUID, + name VARCHAR(255) NOT NULL, + namespace_uuid UUID REFERENCES namespaces(uuid), + type VARCHAR(64), + is_primary BOOLEAN DEFAULT FALSE, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + UNIQUE (namespace_uuid, name) +); + +CREATE INDEX dataset_symlinks_dataset_uuid on dataset_symlinks (dataset_uuid); + +INSERT INTO dataset_symlinks (dataset_uuid, name, namespace_uuid, is_primary, created_at, updated_at) +SELECT d.uuid, d.name, d.namespace_uuid, TRUE, d.created_at, d.updated_at FROM datasets d; + +DROP TYPE IF EXISTS DATASET_NAME; +CREATE TYPE DATASET_NAME AS ( + namespace VARCHAR(255), + name VARCHAR(255) +); + diff --git a/api/src/main/resources/marquez/db/migration/V48__column_level_lineage.sql b/api/src/main/resources/marquez/db/migration/V49__column_level_lineage.sql similarity index 100% rename from api/src/main/resources/marquez/db/migration/V48__column_level_lineage.sql rename to api/src/main/resources/marquez/db/migration/V49__column_level_lineage.sql diff --git a/api/src/test/java/marquez/OpenLineageIntegrationTest.java b/api/src/test/java/marquez/OpenLineageIntegrationTest.java index 3bf575c07a..8e3f8ce0a3 100644 --- a/api/src/test/java/marquez/OpenLineageIntegrationTest.java +++ b/api/src/test/java/marquez/OpenLineageIntegrationTest.java @@ -31,6 +31,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -199,11 +200,260 @@ public void testOpenLineageJobHierarchyAirflowIntegration() String dagName = "the_dag"; RunEvent airflowTask1 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); + + RunEvent airflowTask2 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); + + CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); + future.get(5, TimeUnit.SECONDS); + + Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name); + assertThat(job) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name)) + .hasFieldOrPropertyWithValue("parentJobName", dagName); + + Job parentJob = client.getJob(NAMESPACE_NAME, dagName); + assertThat(parentJob) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName)) + .hasFieldOrPropertyWithValue("parentJobName", null); + List runsList = client.listRuns(NAMESPACE_NAME, dagName); + assertThat(runsList).isNotEmpty().hasSize(1); + } + + @Test + public void testOpenLineageJobHierarchyAirflowIntegrationWithParentRunFacet() + throws ExecutionException, InterruptedException, TimeoutException { + OpenLineage ol = new OpenLineage(URI.create("http://openlineage.test.com/")); + ZonedDateTime startOfHour = + Instant.now() + .atZone(LineageTestUtils.LOCAL_ZONE) + .with(ChronoField.MINUTE_OF_HOUR, 0) + .with(ChronoField.SECOND_OF_MINUTE, 0); + ZonedDateTime endOfHour = startOfHour.plusHours(1); + String airflowParentRunId = UUID.randomUUID().toString(); + String task1Name = "task1"; + String task2Name = "task2"; + String dagName = "the_dag"; + RunEvent airflowTask1 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); + + // the older airflow integration reported parentRun instead of parent. We support this as an + // alias for compatibility + RunFacet parent = airflowTask1.getRun().getFacets().getAdditionalProperties().remove("parent"); + airflowTask1.getRun().getFacets().getAdditionalProperties().put("parentRun", parent); + + RunEvent airflowTask2 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); + parent = airflowTask2.getRun().getFacets().getAdditionalProperties().remove("parent"); + airflowTask2.getRun().getFacets().getAdditionalProperties().put("parentRun", parent); + + CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); + future.get(5, TimeUnit.SECONDS); + + Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name); + assertThat(job) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name)) + .hasFieldOrPropertyWithValue("parentJobName", dagName); + + Job parentJob = client.getJob(NAMESPACE_NAME, dagName); + assertThat(parentJob) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName)) + .hasFieldOrPropertyWithValue("parentJobName", null); + List runsList = client.listRuns(NAMESPACE_NAME, dagName); + assertThat(runsList).isNotEmpty().hasSize(1); + } + + @Test + public void testOpenLineageJobHierarchyAirflowIntegrationWithParentAndParentRunFacet() + throws ExecutionException, InterruptedException, TimeoutException { + OpenLineage ol = new OpenLineage(URI.create("http://openlineage.test.com/")); + ZonedDateTime startOfHour = + Instant.now() + .atZone(LineageTestUtils.LOCAL_ZONE) + .with(ChronoField.MINUTE_OF_HOUR, 0) + .with(ChronoField.SECOND_OF_MINUTE, 0); + ZonedDateTime endOfHour = startOfHour.plusHours(1); + String airflowParentRunId = UUID.randomUUID().toString(); + String task1Name = "task1"; + String task2Name = "task2"; + String dagName = "the_dag"; + RunEvent airflowTask1 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); + + // the older airflow integration reported parentRun instead of parent. The new integration + // reports both. They are the same in the airflow integration, but this test verifies we handle + // the "parentRun" field first. + // It would be preferable to prioritize the "parent" field, but it seems Jackson prefers the + // alias first. + RunFacet parent = airflowTask1.getRun().getFacets().getAdditionalProperties().get("parent"); + RunFacet newParent = ol.newRunFacet(); + Map runFacetProps = newParent.getAdditionalProperties(); + runFacetProps.put("run", parent.getAdditionalProperties().get("run")); + runFacetProps.put( + "job", ImmutableMap.of("name", "a_new_dag", "namespace", "incorrect_namespace")); + airflowTask1.getRun().getFacets().getAdditionalProperties().put("parentRun", parent); + airflowTask1.getRun().getFacets().getAdditionalProperties().put("parent", newParent); + + RunEvent airflowTask2 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); + parent = airflowTask2.getRun().getFacets().getAdditionalProperties().get("parent"); + newParent = ol.newRunFacet(); + runFacetProps = newParent.getAdditionalProperties(); + runFacetProps.put("run", parent.getAdditionalProperties().get("run")); + runFacetProps.put( + "job", ImmutableMap.of("name", "a_new_dag", "namespace", "incorrect_namespace")); + airflowTask2.getRun().getFacets().getAdditionalProperties().put("parentRun", parent); + airflowTask2.getRun().getFacets().getAdditionalProperties().put("parent", newParent); + + CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); + future.get(5, TimeUnit.SECONDS); + + Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name); + assertThat(job) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name)) + .hasFieldOrPropertyWithValue("parentJobName", dagName); + + Job parentJob = client.getJob(NAMESPACE_NAME, dagName); + assertThat(parentJob) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName)) + .hasFieldOrPropertyWithValue("parentJobName", null); + List runsList = client.listRuns(NAMESPACE_NAME, dagName); + assertThat(runsList).isNotEmpty().hasSize(1); + } + + @Test + public void testOpenLineageJobHierarchyAirflowIntegrationWithDagNameWithDot() + throws ExecutionException, InterruptedException, TimeoutException { + OpenLineage ol = new OpenLineage(URI.create("http://openlineage.test.com/")); + ZonedDateTime startOfHour = + Instant.now() + .atZone(LineageTestUtils.LOCAL_ZONE) + .with(ChronoField.MINUTE_OF_HOUR, 0) + .with(ChronoField.SECOND_OF_MINUTE, 0); + ZonedDateTime endOfHour = startOfHour.plusHours(1); + String airflowParentRunId = UUID.randomUUID().toString(); + String task1Name = "task1"; + String task2Name = "task2"; + String dagName = "the.dag"; + RunEvent airflowTask1 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); RunEvent airflowTask2 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task2Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); + + CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); + future.get(5, TimeUnit.SECONDS); + + Job job = client.getJob(NAMESPACE_NAME, dagName + "." + task1Name); + assertThat(job) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName + "." + task1Name)) + .hasFieldOrPropertyWithValue("parentJobName", dagName); + + Job parentJob = client.getJob(NAMESPACE_NAME, dagName); + assertThat(parentJob) + .isNotNull() + .hasFieldOrPropertyWithValue("id", new JobId(NAMESPACE_NAME, dagName)) + .hasFieldOrPropertyWithValue("parentJobName", null); + List runsList = client.listRuns(NAMESPACE_NAME, dagName); + assertThat(runsList).isNotEmpty().hasSize(1); + } + + @Test + public void testOpenLineageJobHierarchyAirflowIntegrationWithTaskGroup() + throws ExecutionException, InterruptedException, TimeoutException { + OpenLineage ol = new OpenLineage(URI.create("http://openlineage.test.com/")); + ZonedDateTime startOfHour = + Instant.now() + .atZone(LineageTestUtils.LOCAL_ZONE) + .with(ChronoField.MINUTE_OF_HOUR, 0) + .with(ChronoField.SECOND_OF_MINUTE, 0); + ZonedDateTime endOfHour = startOfHour.plusHours(1); + String airflowParentRunId = UUID.randomUUID().toString(); + String task1Name = "task_group.task1"; + String task2Name = "task_group.task2"; + String dagName = "dag_with_task_group"; + RunEvent airflowTask1 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); + + RunEvent airflowTask2 = + createAirflowRunEvent( + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task2Name, + NAMESPACE_NAME); CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); future.get(5, TimeUnit.SECONDS); @@ -242,13 +492,27 @@ public void testOpenLineageJobHierarchyOldAirflowIntegration() String task1Name = "task1"; String task2Name = "task2"; String dagName = "the_dag"; + + // the old integration also used the fully qualified task name as the parent job name RunEvent airflowTask1 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName + "." + task1Name, + dagName + "." + task1Name, + NAMESPACE_NAME); RunEvent airflowTask2 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task2Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName + "." + task2Name, + dagName + "." + task2Name, + NAMESPACE_NAME); CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); future.get(5, TimeUnit.SECONDS); @@ -291,12 +555,24 @@ public void testOpenLineageJobHierarchyAirflowIntegrationConflictingRunUuid() // two dag runs with different namespaces - should result in two distinct jobs RunEvent airflowTask1 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); String secondNamespace = "another_namespace"; RunEvent airflowTask2 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, secondNamespace); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + secondNamespace); CompletableFuture future = sendAllEvents(airflowTask1, airflowTask2); future.get(5, TimeUnit.SECONDS); @@ -332,7 +608,13 @@ public void testOpenLineageJobHierarchySparkAndAirflow() String dagName = "the_dag"; RunEvent airflowTask1 = createAirflowRunEvent( - ol, startOfHour, endOfHour, airflowParentRunId, task1Name, dagName, NAMESPACE_NAME); + ol, + startOfHour, + endOfHour, + airflowParentRunId, + dagName, + dagName + "." + task1Name, + NAMESPACE_NAME); RunEvent sparkTask = createRunEvent( @@ -340,8 +622,8 @@ public void testOpenLineageJobHierarchySparkAndAirflow() startOfHour, endOfHour, airflowTask1.getRun().getRunId().toString(), - sparkTaskName, dagName + "." + task1Name, + dagName + "." + task1Name + "." + sparkTaskName, Optional.empty(), NAMESPACE_NAME); @@ -609,8 +891,8 @@ private RunEvent createAirflowRunEvent( ZonedDateTime startOfHour, ZonedDateTime endOfHour, String airflowParentRunId, - String taskName, String dagName, + String taskName, String namespace) { RunFacet airflowVersionFacet = ol.newRunFacet(); airflowVersionFacet @@ -622,8 +904,8 @@ private RunEvent createAirflowRunEvent( startOfHour, endOfHour, airflowParentRunId, - taskName, dagName, + taskName, Optional.of(airflowVersionFacet), namespace); } @@ -634,8 +916,8 @@ private RunEvent createRunEvent( ZonedDateTime startOfHour, ZonedDateTime endOfHour, String airflowParentRunId, - String taskName, String dagName, + String taskName, Optional airflowVersionFacet, String namespace) { // The Java SDK requires parent run ids to be a UUID, but the python SDK doesn't. In order to @@ -650,7 +932,7 @@ private RunEvent createRunEvent( "run", ImmutableMap.of("runId", airflowParentRunId), "job", - ImmutableMap.of("namespace", namespace, "name", dagName + "." + taskName))); + ImmutableMap.of("namespace", namespace, "name", dagName))); RunFacetsBuilder runFacetBuilder = ol.newRunFacetsBuilder() .nominalTime(ol.newNominalTimeRunFacet(startOfHour, endOfHour)) @@ -663,7 +945,7 @@ private RunEvent createRunEvent( .job( ol.newJob( namespace, - dagName + "." + taskName, + taskName, ol.newJobFacetsBuilder() .documentation(ol.newDocumentationJobFacet("the job docs")) .sql(ol.newSQLJobFacet("SELECT * FROM the_table")) diff --git a/api/src/test/java/marquez/db/DatasetDaoTest.java b/api/src/test/java/marquez/db/DatasetDaoTest.java index c1ef2f7bfa..09e61d03d2 100644 --- a/api/src/test/java/marquez/db/DatasetDaoTest.java +++ b/api/src/test/java/marquez/db/DatasetDaoTest.java @@ -70,6 +70,7 @@ public void tearDown(Jdbi jdbi) { handle.execute("DELETE FROM dataset_versions_field_mapping"); handle.execute("DELETE FROM stream_versions"); handle.execute("DELETE FROM dataset_versions"); + handle.execute("DELETE FROM dataset_symlinks"); handle.execute("UPDATE runs SET start_run_state_uuid=NULL, end_run_state_uuid=NULL"); handle.execute("DELETE FROM run_states"); handle.execute("DELETE FROM runs"); @@ -195,6 +196,33 @@ public void testGetDatasetWithDatasetMarkedDeleted() { assertThat(datasetDao.findWithTags(NAMESPACE, DATASET).get().isDeleted()).isTrue(); } + @Test + public void testGetDatasetBySymlink() { + createLineageRow( + openLineageDao, + "aJob", + "COMPLETE", + jobFacet, + Collections.emptyList(), + Collections.singletonList( + new Dataset( + NAMESPACE, + DATASET, + LineageEvent.DatasetFacets.builder() + .symlinks( + new LineageEvent.DatasetSymlinkFacet( + PRODUCER_URL, + SCHEMA_URL, + Collections.singletonList( + new LineageEvent.SymlinkIdentifier( + "symlinkNamespace", "symlinkName", "type")))) + .build()))); + + // verify dataset is returned by its name and symlink name + assertThat(datasetDao.findDatasetByName(NAMESPACE, DATASET)).isPresent(); + assertThat(datasetDao.findDatasetByName("symlinkNamespace", "symlinkName")).isPresent(); + } + @Test public void testGetDatasetWithMultipleVersions() { createLineageRow( diff --git a/api/src/test/java/marquez/db/LineageDaoTest.java b/api/src/test/java/marquez/db/LineageDaoTest.java index 07b2b9ee7c..0a0becbb9d 100644 --- a/api/src/test/java/marquez/db/LineageDaoTest.java +++ b/api/src/test/java/marquez/db/LineageDaoTest.java @@ -15,6 +15,7 @@ import com.google.common.base.Functions; import java.sql.SQLException; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -84,6 +85,7 @@ public void tearDown(Jdbi jdbi) { handle.execute("DELETE FROM runs_input_mapping"); handle.execute("DELETE FROM dataset_versions_field_mapping"); handle.execute("DELETE FROM dataset_versions"); + handle.execute("DELETE FROM dataset_symlinks"); handle.execute("UPDATE runs SET start_run_state_uuid=NULL, end_run_state_uuid=NULL"); handle.execute("DELETE FROM run_states"); handle.execute("DELETE FROM runs"); @@ -220,7 +222,7 @@ public void testGetLineageForSymlinkedJob() throws SQLException { .upsertJob( UUID.randomUUID(), JobType.valueOf(writeJob.getJob().getType()), - writeJob.getJob().getCreatedAt(), + Instant.now(), namespaceRow.getUuid(), writeJob.getJob().getNamespaceName(), symlinkTargetJobName, @@ -233,7 +235,7 @@ public void testGetLineageForSymlinkedJob() throws SQLException { .upsertJob( writeJob.getJob().getUuid(), JobType.valueOf(writeJob.getJob().getType()), - writeJob.getJob().getCreatedAt(), + Instant.now(), namespaceRow.getUuid(), writeJob.getJob().getNamespaceName(), writeJob.getJob().getName(), diff --git a/api/src/test/java/marquez/db/OpenLineageDaoTest.java b/api/src/test/java/marquez/db/OpenLineageDaoTest.java index da2d9ec023..e6993e7656 100644 --- a/api/src/test/java/marquez/db/OpenLineageDaoTest.java +++ b/api/src/test/java/marquez/db/OpenLineageDaoTest.java @@ -45,6 +45,8 @@ class OpenLineageDaoTest { public static final String TRANSFORMATION_DESCRIPTION = "transformation_description"; private static OpenLineageDao dao; + private static DatasetSymlinkDao symlinkDao; + private static NamespaceDao namespaceDao; private static DatasetFieldDao datasetFieldDao; private final DatasetFacets datasetFacets = LineageTestUtils.newDatasetFacet( @@ -53,6 +55,8 @@ class OpenLineageDaoTest { @BeforeAll public static void setUpOnce(Jdbi jdbi) { dao = jdbi.onDemand(OpenLineageDao.class); + symlinkDao = jdbi.onDemand(DatasetSymlinkDao.class); + namespaceDao = jdbi.onDemand(NamespaceDao.class); datasetFieldDao = jdbi.onDemand(DatasetFieldDao.class); } @@ -281,6 +285,56 @@ void testUpsertColumnLineageData() { .isEqualTo(writeJob2.getOutputs().get().get(0).getDatasetVersionRow()); } + @Test + void testUpdateMarquezModelDatasetWithSymlinks() { + Dataset dataset = + new Dataset( + LineageTestUtils.NAMESPACE, + DATASET_NAME, + LineageEvent.DatasetFacets.builder() + .symlinks( + new LineageEvent.DatasetSymlinkFacet( + PRODUCER_URL, + SCHEMA_URL, + Collections.singletonList( + new LineageEvent.SymlinkIdentifier( + "symlinkNamespace", "symlinkName", "some-type")))) + .build()); + + JobFacet jobFacet = new JobFacet(null, null, null, LineageTestUtils.EMPTY_MAP); + UpdateLineageRow writeJob = + LineageTestUtils.createLineageRow( + dao, WRITE_JOB_NAME, "COMPLETE", jobFacet, Arrays.asList(), Arrays.asList(dataset)); + + UpdateLineageRow readJob = + LineageTestUtils.createLineageRow( + dao, + WRITE_JOB_NAME, + "COMPLETE", + jobFacet, + Arrays.asList( + new Dataset( + "symlinkNamespace", + "symlinkName", + LineageEvent.DatasetFacets.builder().build())), + Arrays.asList()); + + // make sure writeJob output dataset and readJob input dataset are the same (have the same uuid) + assertThat(writeJob.getOutputs()).isPresent().get().asList().size().isEqualTo(1); + assertThat(writeJob.getOutputs().get().get(0).getDatasetRow().getUuid()) + .isEqualTo(readJob.getInputs().get().get(0).getDatasetRow().getUuid()); + // make sure symlink is stored with type in dataset_symlinks table + assertThat( + symlinkDao + .findDatasetSymlinkByNamespaceUuidAndName( + namespaceDao.findNamespaceByName("symlinkNamespace").get().getUuid(), + "symlinkName") + .get() + .getType() + .get()) + .isEqualTo("some-type"); + } + /** * When reading a new dataset, a version is created and the dataset's current version is updated * immediately. @@ -334,6 +388,7 @@ void testUpdateMarquezModelWithNonMatchingReadSchema() { this.datasetFacets.getLifecycleStateChange(), this.datasetFacets.getDataSource(), this.datasetFacets.getColumnLineage(), + null, this.datasetFacets.getDescription(), this.datasetFacets.getAdditionalFacets()); UpdateLineageRow readJob = diff --git a/api/src/test/java/marquez/db/RunDaoTest.java b/api/src/test/java/marquez/db/RunDaoTest.java index 87b0d79925..8207f3ad5e 100644 --- a/api/src/test/java/marquez/db/RunDaoTest.java +++ b/api/src/test/java/marquez/db/RunDaoTest.java @@ -69,6 +69,7 @@ public void tearDown(Jdbi jdbi) { handle.execute("DELETE FROM datasets_tag_mapping"); handle.execute("DELETE FROM dataset_versions_field_mapping"); handle.execute("DELETE FROM dataset_versions"); + handle.execute("DELETE FROM dataset_symlinks"); handle.execute("UPDATE runs SET start_run_state_uuid=NULL, end_run_state_uuid=NULL"); handle.execute("DELETE FROM run_states"); handle.execute("DELETE FROM runs"); diff --git a/api/src/test/java/marquez/service/LineageServiceTest.java b/api/src/test/java/marquez/service/LineageServiceTest.java index 86c6a4252c..2879640d85 100644 --- a/api/src/test/java/marquez/service/LineageServiceTest.java +++ b/api/src/test/java/marquez/service/LineageServiceTest.java @@ -88,6 +88,7 @@ public void tearDown(Jdbi jdbi) { handle.execute("DELETE FROM dataset_versions_field_mapping"); handle.execute("DELETE FROM stream_versions"); handle.execute("DELETE FROM dataset_versions"); + handle.execute("DELETE FROM dataset_symlinks"); handle.execute("UPDATE runs SET start_run_state_uuid=NULL, end_run_state_uuid=NULL"); handle.execute("DELETE FROM run_states"); handle.execute("DELETE FROM runs"); diff --git a/api/src/test/java/marquez/service/OpenLineageServiceIntegrationTest.java b/api/src/test/java/marquez/service/OpenLineageServiceIntegrationTest.java index 103c749bce..f6eaf51eeb 100644 --- a/api/src/test/java/marquez/service/OpenLineageServiceIntegrationTest.java +++ b/api/src/test/java/marquez/service/OpenLineageServiceIntegrationTest.java @@ -45,6 +45,7 @@ import marquez.jdbi.MarquezJdbiExternalPostgresExtension; import marquez.service.RunTransitionListener.JobInputUpdate; import marquez.service.RunTransitionListener.JobOutputUpdate; +import marquez.service.RunTransitionListener.RunTransition; import marquez.service.models.Dataset; import marquez.service.models.Job; import marquez.service.models.LineageEvent; @@ -79,6 +80,7 @@ public class OpenLineageServiceIntegrationTest { private DatasetVersionDao datasetVersionDao; private ArgumentCaptor runInputListener; private ArgumentCaptor runOutputListener; + private ArgumentCaptor runTransitionListener; private OpenLineageService lineageService; public static String EVENT_REQUIRED_ONLY = "open_lineage/event_required_only.json"; @@ -145,6 +147,8 @@ public void setup(Jdbi jdbi) throws SQLException { doNothing().when(runService).notify(runInputListener.capture()); runOutputListener = ArgumentCaptor.forClass(JobOutputUpdate.class); doNothing().when(runService).notify(runOutputListener.capture()); + runTransitionListener = ArgumentCaptor.forClass(RunTransition.class); + doNothing().when(runService).notify(runTransitionListener.capture()); lineageService = new OpenLineageService(openLineageDao, runService); datasetDao = jdbi.onDemand(DatasetDao.class); @@ -243,6 +247,19 @@ public void testRunListenerOutput(List uris, ExpectedResults expectedResult } } + @ParameterizedTest + @MethodSource("getData") + public void testRunTransition(List uris, ExpectedResults expectedResults) { + initEvents(uris); + + if (expectedResults.inputEventCount > 0) { + Assertions.assertEquals( + uris.size(), + runTransitionListener.getAllValues().size(), + "RunTransition happens once for each run"); + } + } + @ParameterizedTest @MethodSource({"getData"}) public void serviceCalls(List uris, ExpectedResults expectedResults) { diff --git a/build.gradle b/build.gradle index 49996bd3bd..9476b6cd20 100644 --- a/build.gradle +++ b/build.gradle @@ -21,7 +21,7 @@ buildscript { dependencies { classpath 'com.adarshr:gradle-test-logger-plugin:3.2.0' classpath 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2' - classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.7.2' + classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.11.0' } } @@ -54,15 +54,15 @@ subprojects { } ext { - assertjVersion = '3.22.0' - dropwizardVersion = '2.1.1' + assertjVersion = '3.23.1' + dropwizardVersion = '2.1.2' jacocoVersion = '0.8.8' - junit5Version = '5.8.2' + junit5Version = '5.9.1' lombokVersion = '1.18.24' - mockitoVersion = '4.5.1' - openlineageVersion = '0.13.0' + mockitoVersion = '4.8.0' + openlineageVersion = '0.14.1' slf4jVersion = '1.7.36' - postgresqlVersion = '42.3.7' + postgresqlVersion = '42.5.0' isReleaseVersion = !version.endsWith('SNAPSHOT') } diff --git a/chart/values.yaml b/chart/values.yaml index 334e6f14b5..14e9269be0 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -107,7 +107,7 @@ postgresql: ## @param image.tag PostgreSQL image tag (immutable tags are recommended) ## image: - tag: 0.26.0 + tag: 12.1.0 ## Authentication parameters ## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#setting-the-root-password-on-first-run ## ref: https://github.com/bitnami/bitnami-docker-postgresql/blob/master/README.md#creating-a-database-on-first-run diff --git a/clients/java/build.gradle b/clients/java/build.gradle index c056245f41..7fea543a13 100644 --- a/clients/java/build.gradle +++ b/clients/java/build.gradle @@ -22,8 +22,9 @@ plugins { dependencies { implementation "io.dropwizard:dropwizard-jackson:${dropwizardVersion}" implementation "org.slf4j:slf4j-api:${slf4jVersion}" + implementation 'org.apache.commons:commons-lang3:3.12.0' implementation 'org.apache.httpcomponents:httpclient:4.5.13' - implementation 'org.apache.maven:maven-archiver:3.5.2' + implementation 'org.apache.maven:maven-archiver:3.6.0' testImplementation "org.slf4j:slf4j-simple:${slf4jVersion}" } diff --git a/clients/java/src/main/java/marquez/client/MarquezPathV1.java b/clients/java/src/main/java/marquez/client/MarquezPathV1.java index 2c679ac548..11f2eaf0e3 100644 --- a/clients/java/src/main/java/marquez/client/MarquezPathV1.java +++ b/clients/java/src/main/java/marquez/client/MarquezPathV1.java @@ -16,7 +16,7 @@ import javax.annotation.Nullable; import lombok.NonNull; import marquez.client.models.RunState; -import org.apache.maven.shared.utils.StringUtils; +import org.apache.commons.lang3.StringUtils; class MarquezPathV1 { @VisibleForTesting static final String BASE_PATH = "/api/v1"; diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 3e279e97cf..5320c72cc9 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -12,14 +12,8 @@ if [[ -z "${MARQUEZ_CONFIG}" ]]; then echo "WARNING 'MARQUEZ_CONFIG' not set, using development configuration." fi -if [[ -z "${MARQUEZ_VERSION}" ]]; then - MARQUEZ_VERSION='*' - echo "WARNING 'MARQUEZ_VERSION' not set. Running could fail if directory contains multiple jar versions." -fi - - # Adjust java options for the http server JAVA_OPTS="${JAVA_OPTS} -Duser.timezone=UTC -Dlog4j2.formatMsgNoLookups=true" # Start http server with java options and configuration -java ${JAVA_OPTS} -jar marquez-${MARQUEZ_VERSION}.jar server ${MARQUEZ_CONFIG} +java ${JAVA_OPTS} -jar marquez-*.jar server ${MARQUEZ_CONFIG} diff --git a/docker/up.sh b/docker/up.sh index facd6533a3..66df5222af 100755 --- a/docker/up.sh +++ b/docker/up.sh @@ -5,7 +5,8 @@ set -e -SCRIPTDIR=$(dirname $0) +VERSION=0.26.0 +DOCKER_DIR=$(dirname $0) title() { echo -e "\033[1m${1}\033[0m" @@ -35,7 +36,7 @@ usage() { echo " -a, --api-port int api port (default: 5000)" echo " -m, --api-admin-port int api admin port (default: 5001)" echo " -w, --web-port int web port (default: 3000)" - echo " -t, --tag string image tag (default: latest)" + echo " -t, --tag string image tag (default: ${VERSION})" echo title "FLAGS:" echo " -b, --build build images from source" @@ -55,7 +56,7 @@ args="-V --force-recreate --remove-orphans" API_PORT=5000 API_ADMIN_PORT=5001 WEB_PORT=3000 -TAG=0.19.0 +TAG=${VERSION} while [ $# -gt 0 ]; do case $1 in -a|'--api-port') @@ -107,6 +108,6 @@ if [[ "${SEED}" = "true" ]]; then compose_files+=" -f docker-compose.seed.yml" fi -$SCRIPTDIR/volumes.sh marquez +${DOCKER_DIR}/volumes.sh marquez -API_PORT=${API_PORT} API_ADMIN_PORT=${API_ADMIN_PORT} WEB_PORT=${WEB_PORT} TAG="${TAG}" docker-compose $compose_files up $args +API_PORT=${API_PORT} API_ADMIN_PORT=${API_ADMIN_PORT} WEB_PORT=${WEB_PORT} TAG=${TAG} docker-compose $compose_files up $args diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 41d9927a4d..249e5832f0 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index aa991fceae..ae04661ee7 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 1b6c787337..a69d9cb6c2 100755 --- a/gradlew +++ b/gradlew @@ -205,6 +205,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index ac1b06f938..53a6b238d4 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,7 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +40,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -75,13 +75,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/new-version.sh b/new-version.sh index 6b01ec06b6..c805a9c022 100755 --- a/new-version.sh +++ b/new-version.sh @@ -130,8 +130,8 @@ sed -i "" "s/^version:.*/version: ${RELEASE_VERSION}/g" ./chart/Chart.yaml sed -i "" "s/tag:.*/tag: ${RELEASE_VERSION}/g" ./chart/values.yaml # (3) Bump version in scripts -sed -i "" "s/TAG=\d.*/TAG=${RELEASE_VERSION}/g" ./docker/up.sh -sed -i "" "s/TAG=\d.*/TAG=${RELEASE_VERSION}/g" .env.example +sed -i "" "s/VERSION=.*/VERSION=${RELEASE_VERSION}/g" ./docker/up.sh +sed -i "" "s/TAG=.*/TAG=${RELEASE_VERSION}/g" .env.example # (4) Bump version in docs sed -i "" "s/^ version:.*/ version: ${RELEASE_VERSION}/g" ./spec/openapi.yml diff --git a/spec/openapi.yml b/spec/openapi.yml index ba4e3af143..111c61655f 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -339,14 +339,14 @@ paths: summary: Soft deletes job. description: Soft deletes job. It will be un-deleted if new OpenLineage event containing this job comes. tags: - - Datasets + - Jobs responses: '200': description: OK content: application/json: schema: - $ref: '#/components/schemas/Dataset' + $ref: '#/components/schemas/Job' /namespaces/{namespace}/jobs: parameters: diff --git a/web/package-lock.json b/web/package-lock.json index b9c5a8c20c..19f924498c 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -39,7 +39,7 @@ "dagre": "^0.8.5", "http-proxy-middleware": "^0.20.0", "lodash": "^4.17.21", - "moment": "^2.29.2", + "moment": "^2.29.4", "postcss-loader": "^3.0.0", "postcss-modules-values": "^2.0.0", "react": "^16.8.0", @@ -1603,6 +1603,58 @@ "node": ">=8" } }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", + "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.15", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.15.tgz", + "integrity": "sha512-oWZNOULl+UbhsgB51uuZzglikfIKSUBO/M9W2OfEjn7cmqoAiCgmv9lyACTUacZwBz0ITnJ2NqjU8Tx0DHL88g==", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@material-ui/core": { "version": "4.12.3", "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.12.3.tgz", @@ -11889,9 +11941,9 @@ } }, "node_modules/moment": { - "version": "2.29.2", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz", - "integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==", + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==", "engines": { "node": "*" } @@ -14907,13 +14959,13 @@ } }, "node_modules/terser": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.12.0.tgz", - "integrity": "sha512-R3AUhNBGWiFc77HXag+1fXpAxTAFRQTJemlJKjAgD9r8xXTpjNKqIXwHM/o7Rh+O0kUJtS3WQVdBeMKFk5sw9A==", + "version": "5.15.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.0.tgz", + "integrity": "sha512-L1BJiXVmheAQQy+as0oF3Pwtlo4s3Wi1X2zNZ2NxOB4wx9bdS9Vk67XQENLFdLYGCK/Z2di53mTj/hBafR+dTA==", "dependencies": { + "@jridgewell/source-map": "^0.3.2", "acorn": "^8.5.0", "commander": "^2.20.0", - "source-map": "~0.7.2", "source-map-support": "~0.5.20" }, "bin": { @@ -14981,14 +15033,6 @@ "randombytes": "^2.1.0" } }, - "node_modules/terser/node_modules/source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==", - "engines": { - "node": ">= 8" - } - }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -17671,6 +17715,49 @@ } } }, + "@jridgewell/gen-mapping": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.2.tgz", + "integrity": "sha512-mh65xKQAzI6iBcFzwv28KVWSmCkdRBWoOh+bYQGW3+6OZvbbN3TqMGo5hqYxQniRcH9F2VZIoJCm4pa3BPDK/A==", + "requires": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==" + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==" + }, + "@jridgewell/source-map": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", + "integrity": "sha512-m7O9o2uR8k2ObDysZYzdfhb08VuEml5oWGiosa1VdaPZ/A6QyPkAJuwN0Q1lhULOf6B7MtQmHENS743hWtCrgw==", + "requires": { + "@jridgewell/gen-mapping": "^0.3.0", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" + }, + "@jridgewell/trace-mapping": { + "version": "0.3.15", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.15.tgz", + "integrity": "sha512-oWZNOULl+UbhsgB51uuZzglikfIKSUBO/M9W2OfEjn7cmqoAiCgmv9lyACTUacZwBz0ITnJ2NqjU8Tx0DHL88g==", + "requires": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "@material-ui/core": { "version": "4.12.3", "resolved": "https://registry.npmjs.org/@material-ui/core/-/core-4.12.3.tgz", @@ -25783,9 +25870,9 @@ } }, "moment": { - "version": "2.29.2", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.2.tgz", - "integrity": "sha512-UgzG4rvxYpN15jgCmVJwac49h9ly9NurikMWGPdVxm8GZD6XjkKPxDTjQQ43gtGgnV3X0cAyWDdP2Wexoquifg==" + "version": "2.29.4", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.4.tgz", + "integrity": "sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==" }, "moo": { "version": "0.4.3", @@ -28158,21 +28245,14 @@ } }, "terser": { - "version": "5.12.0", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.12.0.tgz", - "integrity": "sha512-R3AUhNBGWiFc77HXag+1fXpAxTAFRQTJemlJKjAgD9r8xXTpjNKqIXwHM/o7Rh+O0kUJtS3WQVdBeMKFk5sw9A==", + "version": "5.15.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.15.0.tgz", + "integrity": "sha512-L1BJiXVmheAQQy+as0oF3Pwtlo4s3Wi1X2zNZ2NxOB4wx9bdS9Vk67XQENLFdLYGCK/Z2di53mTj/hBafR+dTA==", "requires": { + "@jridgewell/source-map": "^0.3.2", "acorn": "^8.5.0", "commander": "^2.20.0", - "source-map": "~0.7.2", "source-map-support": "~0.5.20" - }, - "dependencies": { - "source-map": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.3.tgz", - "integrity": "sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ==" - } } }, "terser-webpack-plugin": { diff --git a/web/package.json b/web/package.json index 36816676f0..b4a17586b3 100644 --- a/web/package.json +++ b/web/package.json @@ -47,7 +47,7 @@ "dagre": "^0.8.5", "http-proxy-middleware": "^0.20.0", "lodash": "^4.17.21", - "moment": "^2.29.2", + "moment": "^2.29.4", "postcss-loader": "^3.0.0", "postcss-modules-values": "^2.0.0", "react": "^16.8.0",