From f79b2c958ab279162bfcab1932393aeb02a85632 Mon Sep 17 00:00:00 2001 From: Kerem Sahin Date: Thu, 11 Jun 2020 01:04:52 -0700 Subject: [PATCH] fix(ingestion): Fix sample MCE for data process --- .../metadata/builders/search/DataProcessIndexBuilder.java | 5 +---- metadata-ingestion/mce-cli/bootstrap_mce.dat | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/metadata-builders/src/main/java/com/linkedin/metadata/builders/search/DataProcessIndexBuilder.java b/metadata-builders/src/main/java/com/linkedin/metadata/builders/search/DataProcessIndexBuilder.java index 3b1c1d7eb42f7..720ccf7939dee 100644 --- a/metadata-builders/src/main/java/com/linkedin/metadata/builders/search/DataProcessIndexBuilder.java +++ b/metadata-builders/src/main/java/com/linkedin/metadata/builders/search/DataProcessIndexBuilder.java @@ -29,9 +29,6 @@ public List getDocumentsToUpdate(@Nonnull RecordTemplate sn @Nonnull @Override public Class getDocumentType() { - throw new UnsupportedOperationException( - String.format("%s doesn't support this feature yet,", - this.getClass().getName()) - ); + return DataProcessDocument.class; } } diff --git a/metadata-ingestion/mce-cli/bootstrap_mce.dat b/metadata-ingestion/mce-cli/bootstrap_mce.dat index 99c8de8b7f399..3771eefc9ed4b 100644 --- a/metadata-ingestion/mce-cli/bootstrap_mce.dat +++ b/metadata-ingestion/mce-cli/bootstrap_mce.dat @@ -3,4 +3,4 @@ {"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleKafkaSchema","platform":"urn:li:dataPlatform:kafka","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None} {"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.dataset.UpstreamLineage", {"upstreams":[{"auditStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"dataset":"urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)","type":"TRANSFORMED"}]}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleHdfsSchema","platform":"urn:li:dataPlatform:hdfs","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None} {"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.dataset.UpstreamLineage", {"upstreams":[{"auditStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"dataset":"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)","type":"TRANSFORMED"}]}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleHiveSchema","platform":"urn:li:dataPlatform:hive","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None} -{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DataProcessSnapshot", {"urn": "urn:li:dataprocess:(sqoop,DEMO,PROD)", "aspects": [{"owners": [{"owner": "urn:li:corpuser:datahub", "type": "DATAOWNER"}], "lastModified": {"time": 0, "actor": "urn:li:corpuser:datahub"}}, ( "com.linkedin.pegasus2avro.dataprocess.DataProcessInfo", { "outputs": [ "urn:li:dataset:(urn:li:dataPlatform:cassandra,barEarth,DEV)", "urn:li:dataset:(urn:li:dataPlatform:cassandra,barMars,DEV)" ], "inputs": [ "urn:li:dataset:(urn:li:dataPlatform:hbase,barSky,PROD)", "urn:li:dataset:(urn:li:dataPlatform:hbase,barOcean,PROD)" ] } )]}), "proposedDelta": None} +{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DataProcessSnapshot", {"urn": "urn:li:dataProcess:(sqoop,DEMO,PROD)", "aspects": [{"owners":[{"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:datahub"}},{"inputs":["urn:li:dataset:(urn:li:dataPlatform:cassandra,barEarth,DEV)","urn:li:dataset:(urn:li:dataPlatform:cassandra,barMars,DEV)"],"outputs":["urn:li:dataset:(urn:li:dataPlatform:hbase,barSky,PROD)","urn:li:dataset:(urn:li:dataPlatform:hbase,barOcean,PROD)"]}]}), "proposedDelta": None}