Skip to content

Commit

Permalink
fix(ingestion): Fix sample MCE for data process
Browse files Browse the repository at this point in the history
  • Loading branch information
Kerem Sahin committed Jun 11, 2020
1 parent 5fc80ae commit f79b2c9
Show file tree
Hide file tree
Showing 2 changed files with 2 additions and 5 deletions.
5 changes: 1 addition & 4 deletions ...builders/src/main/java/com/linkedin/metadata/builders/search/DataProcessIndexBuilder.java
View file
Expand Up @@ -29,9 +29,6 @@ public List<DataProcessDocument> getDocumentsToUpdate(@Nonnull RecordTemplate sn
@Nonnull
@Override
public Class<DataProcessDocument> getDocumentType() {
throw new UnsupportedOperationException(
String.format("%s doesn't support this feature yet,",
this.getClass().getName())
);
return DataProcessDocument.class;
}
}
2 changes: 1 addition & 1 deletion metadata-ingestion/mce-cli/bootstrap_mce.dat
View file
Expand Up @@ -3,4 +3,4 @@
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleKafkaSchema","platform":"urn:li:dataPlatform:kafka","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleKafkaSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Kafka dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.dataset.UpstreamLineage", {"upstreams":[{"auditStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"dataset":"urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)","type":"TRANSFORMED"}]}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleHdfsSchema","platform":"urn:li:dataPlatform:hdfs","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot", {"urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", "aspects": [("com.linkedin.pegasus2avro.common.Ownership", {"owners":[{"owner":"urn:li:corpuser:jdoe","type":"DATAOWNER"}, {"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}),("com.linkedin.pegasus2avro.dataset.UpstreamLineage", {"upstreams":[{"auditStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"dataset":"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)","type":"TRANSFORMED"}]}),("com.linkedin.pegasus2avro.common.InstitutionalMemory", {"elements":[{"url":"https://www.linkedin.com","description":"Sample doc","createStamp":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"}}]}), ("com.linkedin.pegasus2avro.schema.SchemaMetadata", {"schemaName":"SampleHiveSchema","platform":"urn:li:dataPlatform:hive","version":0,"created":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:jdoe"},"hash":"","platformSchema":{"documentSchema":"{\"type\":\"record\",\"name\":\"SampleHiveSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample Hive dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}"},"fields":[{"fieldPath":"field_foo","description":"Foo field description","nativeDataType":"string","type":{"type":{"com.linkedin.pegasus2avro.schema.StringType":{}}}},{"fieldPath":"field_bar","description":"Bar field description","nativeDataType":"boolean","type":{"type":{"com.linkedin.pegasus2avro.schema.BooleanType":{}}}}]})]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DataProcessSnapshot", {"urn": "urn:li:dataprocess:(sqoop,DEMO,PROD)", "aspects": [{"owners": [{"owner": "urn:li:corpuser:datahub", "type": "DATAOWNER"}], "lastModified": {"time": 0, "actor": "urn:li:corpuser:datahub"}}, ( "com.linkedin.pegasus2avro.dataprocess.DataProcessInfo", { "outputs": [ "urn:li:dataset:(urn:li:dataPlatform:cassandra,barEarth,DEV)", "urn:li:dataset:(urn:li:dataPlatform:cassandra,barMars,DEV)" ], "inputs": [ "urn:li:dataset:(urn:li:dataPlatform:hbase,barSky,PROD)", "urn:li:dataset:(urn:li:dataPlatform:hbase,barOcean,PROD)" ] } )]}), "proposedDelta": None}
{"auditHeader": None, "proposedSnapshot": ("com.linkedin.pegasus2avro.metadata.snapshot.DataProcessSnapshot", {"urn": "urn:li:dataProcess:(sqoop,DEMO,PROD)", "aspects": [{"owners":[{"owner":"urn:li:corpuser:datahub","type":"DATAOWNER"}],"lastModified":{"time":1581407189000,"actor":"urn:li:corpuser:datahub"}},{"inputs":["urn:li:dataset:(urn:li:dataPlatform:cassandra,barEarth,DEV)","urn:li:dataset:(urn:li:dataPlatform:cassandra,barMars,DEV)"],"outputs":["urn:li:dataset:(urn:li:dataPlatform:hbase,barSky,PROD)","urn:li:dataset:(urn:li:dataPlatform:hbase,barOcean,PROD)"]}]}), "proposedDelta": None}

0 comments on commit f79b2c9

Please sign in to comment.