Permalink
Browse files

PDSC to AVSC translator supports namespace override based on cmd option

RB=1220188
BUG=SI-4795
G=si-dev
R=xma,mlamure,mnchen
A=xma,mnchen
  • Loading branch information...
seliang committed Feb 23, 2018
1 parent 3b4152a commit 274b70294ae76a42a7696f0727e245e511fdcb5f
@@ -1,5 +1,7 @@
19.0.4
------
(RB=1220188)
PDSC to AVSC translator supports namespace override based on cmd option.
(RB=1230599)
Replace dependency on antlr with antlr-runtime.
(RB=1230938)
@@ -45,6 +45,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.linkedin.data.avro.SchemaTranslator.AVRO_PREFIX;
/**
* Generate Avro avsc files from {@link RecordDataSchema}s.
@@ -62,6 +64,7 @@
public class AvroSchemaGenerator extends AbstractGenerator
{
public static final String GENERATOR_AVRO_TRANSLATE_OPTIONAL_DEFAULT = "generator.avro.optional.default";
public static final String GENERATOR_AVRO_NAMESPACE_OVERRIDE = "generator.avro.namespace.override";
private static final Logger _log = LoggerFactory.getLogger(AvroSchemaGenerator.class);
@@ -109,6 +112,7 @@ public static void main(String[] args) throws IOException
run(System.getProperty(GENERATOR_RESOLVER_PATH),
System.getProperty(GENERATOR_AVRO_TRANSLATE_OPTIONAL_DEFAULT),
Boolean.parseBoolean(System.getProperty(GENERATOR_AVRO_NAMESPACE_OVERRIDE)),
args[0],
Arrays.copyOfRange(args, 1, args.length));
}
@@ -119,7 +123,7 @@ public AvroSchemaGenerator(Config config)
_config = config;
}
public static void run(String resolverPath, String optionalDefault, String targetDirectoryPath, String[] sources) throws IOException
public static void run(String resolverPath, String optionalDefault, boolean overrideNamespace, String targetDirectoryPath, String[] sources) throws IOException
{
final AvroSchemaGenerator generator = new AvroSchemaGenerator(new Config(resolverPath));
@@ -128,7 +132,11 @@ public static void run(String resolverPath, String optionalDefault, String targe
final OptionalDefaultMode optionalDefaultMode = OptionalDefaultMode.valueOf(optionalDefault.toUpperCase());
generator.getDataToAvroSchemaTranslationOptions().setOptionalDefaultMode(optionalDefaultMode);
}
generator.getDataToAvroSchemaTranslationOptions().setOverrideNamespace(overrideNamespace);
if (overrideNamespace)
{
targetDirectoryPath += "/" + AVRO_PREFIX;
}
generator.generate(targetDirectoryPath, sources);
}

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -123,6 +123,30 @@ public DataToAvroSchemaTranslationOptions setOptionalDefaultMode(OptionalDefault
return this;
}
/**
* Set namespace override option.
* If the overrideNamespace is true, the namespace in avcs generated by pdsc will be prefixed with AVRO_PREFIX.
*
* @param overrideNamespace If the overrideNamespace is true,
* the namespace in avcs generated by pdsc will be prefixed with AVRO_PREFIX.
* @return {@code this}.
*/
public DataToAvroSchemaTranslationOptions setOverrideNamespace(boolean overrideNamespace)
{
_overrideNamespace = overrideNamespace;
return this;
}
/**
* return override namespace option.
*
* @return override namespace option.
*/
public boolean isOverrideNamespace()
{
return _overrideNamespace;
}
/**
* Return how an optional field and associated default value should be translated.
*
@@ -175,4 +199,5 @@ public EmbedSchemaMode getEmbeddedSchema()
private OptionalDefaultMode _optionalDefaultMode;
private JsonBuilder.Pretty _pretty;
private EmbedSchemaMode _embedSchemaMode;
private boolean _overrideNamespace = false;
}
@@ -23,6 +23,8 @@
import com.linkedin.data.schema.DataSchema;
import com.linkedin.data.schema.DataSchemaConstants;
import com.linkedin.data.schema.JsonBuilder;
import com.linkedin.data.schema.Named;
import com.linkedin.data.schema.NamedDataSchema;
import com.linkedin.data.schema.RecordDataSchema;
import com.linkedin.data.schema.SchemaToJsonEncoder;
import com.linkedin.data.schema.UnionDataSchema;
@@ -37,10 +39,10 @@
import java.util.Set;
import java.util.stream.Collectors;
import static com.linkedin.data.avro.SchemaTranslator.AVRO_PREFIX;
import static com.linkedin.data.schema.DataSchemaConstants.DEFAULT_KEY;
import static com.linkedin.data.schema.DataSchemaConstants.TYPE_KEY;
/**
* Serializes and outputs {@link DataSchema}s in
* Avro-compliant schema in JSON representation.
@@ -385,6 +387,32 @@ protected void encodeFieldProperties(RecordDataSchema.Field field) throws IOExce
_builder.writeProperties(Maps.filterKeys(field.getProperties(), property -> !RESERVED_DATA_PROPERTIES.contains(property)));
}
/**
* Encode namespace in the {@link Named}.
*
* This method encodes the namespace fields.
* If the override namespace option is true, the namespace will be prefixed with AVRO_PREFIX.
*
* @param schema provides the {@link NamedDataSchema}.
*/
@Override
protected String encodeNamespace(Named schema)
{
String namespace = schema.getNamespace();
if (_options.isOverrideNamespace())
{
if (!namespace.isEmpty())
{
namespace = AVRO_PREFIX + "." + namespace;
}
else {
namespace = AVRO_PREFIX;
}
}
return namespace;
}
@Override
protected void encodeField(RecordDataSchema.Field field) throws IOException
{
super.encodeField(field);
@@ -51,6 +51,7 @@
public static final String SCHEMA_PROPERTY = "schema";
public static final String OPTIONAL_DEFAULT_MODE_PROPERTY = "optionalDefaultMode";
public static final String AVRO_FILE_EXTENSION = ".avsc";
public static final String AVRO_PREFIX = "avro";
public static final String CONTAINER_RECORD_DISCRIMINATOR_ENUM_SUFFIX = "Discriminator";
public static final String TRANSLATED_UNION_MEMBER_PROPERTY = "translatedUnionMember";
@@ -1883,6 +1883,37 @@ public void testEmbeddingSchemaWithDataProperty(String schemaText, String expect
assertEquals(avroSchemaDataMap, expectedDataMap);
}
@DataProvider
public Object[][] schemaWithNamespaceOverride()
{
return new Object[][]
{
{
// no namespace
"{ \"type\" : \"record\", \"name\" : \"foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : \"int\" } ]}",
"{ \"type\" : \"record\", \"name\" : \"foo\", \"namespace\" : \"avro\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : \"int\" } ] }"
},
{
// namespace inside name
"{ \"type\" : \"record\", \"name\" : \"x.y.foo\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : \"int\" } ]}",
"{ \"type\" : \"record\", \"name\" : \"foo\", \"namespace\" : \"avro.x.y\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : \"int\" } ] }"
},
{
// exist namespace
"{ \"type\" : \"record\", \"name\" : \"foo\", \"namespace\" : \"x.y\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : \"int\" } ]}",
"{ \"type\" : \"record\", \"name\" : \"foo\", \"namespace\" : \"avro.x.y\", \"fields\" : [ { \"name\" : \"bar\", \"type\" : \"int\" } ] }"
}
};
}
@Test(dataProvider = "schemaWithNamespaceOverride")
public void testSchemaWithNamespaceOverride(String schemaText, String expected) throws IOException
{
DataToAvroSchemaTranslationOptions options = new DataToAvroSchemaTranslationOptions(JsonBuilder.Pretty.SPACES).setOverrideNamespace(true);
String avroSchemaJson = SchemaTranslator.dataToAvroSchemaJson(TestUtil.dataSchemaFromString(schemaText), options);
assertEquals(avroSchemaJson, expected);
}
@DataProvider
public Object[][] fromAvroSchemaData()
{
@@ -225,7 +225,7 @@ protected void encodeNamed(NamedDataSchema schema, TypeRepresentation representa
_builder.writeStartObject();
_builder.writeStringField(TYPE_KEY, schema.getType().toString().toLowerCase(), true);
encodeName(NAME_KEY, schema);
encodeName(schema);
final String packageName = schema.getPackage();
if (packageName != null && !_currentPackage.equals(packageName))
{
@@ -489,17 +489,16 @@ protected void encodeFieldProperties(RecordDataSchema.Field field) throws IOExce
* It also adds the fully qualified name to the set of names already dumped
* and updates the current namespace.
*
* @param nameKey provides the key used for the name.
* @param schema provides the {@link NamedDataSchema}.
* @throws IOException if there is an error while encoding.
*/
protected void encodeName(String nameKey, Named schema) throws IOException
protected void encodeName(Named schema) throws IOException
{
String fullName = schema.getFullName();
if (fullName.isEmpty() == false)
{
String namespace = schema.getNamespace();
_builder.writeStringField(nameKey, schema.getName(), true);
String namespace = encodeNamespace(schema);
_builder.writeStringField(NAME_KEY, schema.getName(), true);
if (_currentNamespace.equals(namespace) == false)
{
_builder.writeStringField(NAMESPACE_KEY, namespace, true);
@@ -508,6 +507,18 @@ protected void encodeName(String nameKey, Named schema) throws IOException
}
}
/**
* Encode namespace in the {@link Named}.
*
* This method encodes the namespace fields.
*
* @param schema provides the {@link NamedDataSchema}.
*/
protected String encodeNamespace(Named schema)
{
return schema.getNamespace();
}
/**
* Whether to encode the "include" attribute and not encode the included fields.
*

0 comments on commit 274b702

Please sign in to comment.