ATLAS-5310: Export/Import API : When tag attributes types get modified between th…

graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java

@@ -47,6 +47,13 @@ public interface AtlasGraphManagement extends AutoCloseable {
      */
     AtlasEdgeLabel makeEdgeLabel(String label);
 
+    /**
+     * @param propertyName
+     * @param dataType
+     * @return true if the property key exists and has the given data type
+     */
+    boolean propertyKeyHasDataType(String propertyName, Class<?> dataType);
+
     /**
      *  @param propertyKey
      *

graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java

@@ -210,6 +210,15 @@ public AtlasEdgeLabel makeEdgeLabel(String label) {
         return GraphDbObjectFactory.createEdgeLabel(edgeLabel);
     }
 
+    @Override
+    public boolean propertyKeyHasDataType(String propertyName, Class<?> dataType) {
+        checkName(propertyName);
+
+        PropertyKey janusPropertyKey = management.getPropertyKey(propertyName);
+
+        return janusPropertyKey != null && janusPropertyKey.dataType().equals(dataType);
+    }
+
     @Override
     public void deletePropertyKey(String propertyKey) {
         PropertyKey janusPropertyKey = management.getPropertyKey(propertyKey);

repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java

@@ -21,6 +21,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.atlas.ApplicationProperties;
 import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
 import org.apache.atlas.discovery.SearchIndexer;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.ha.HAConfiguration;
@@ -439,6 +440,14 @@ public String createVertexIndex(AtlasGraphManagement management, String property
         if (propertyName != null) {
             AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
 
+            if (propertyKey != null && RequestContext.get().isImportInProgress()
+                    && !management.propertyKeyHasDataType(propertyName, propertyClass)) {
+                LOG.info("Recreating property key {} during import; data type changed to {}", propertyName, propertyClass.getName());
+
+                management.deletePropertyKey(propertyName);
+                propertyKey = null;
+            }
+
             if (propertyKey == null) {
                 propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
 

repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java

@@ -18,7 +18,6 @@
 package org.apache.atlas.repository.impexp;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.atlas.AtlasErrorCode;
 import org.apache.atlas.exception.AtlasBaseException;
 import org.apache.atlas.model.impexp.AtlasImportResult;
 import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
@@ -170,12 +169,11 @@ private void updateCollectionWithDifferingAttributes(List<AtlasStructDef.AtlasAt
             if (relationshipType == null) {
                 difference.add(incoming);
             }
-        } else {
-            if (!existingAttribute.getTypeName().equals(incoming.getTypeName())) {
-                LOG.error("Attribute definition difference found: {}, {}", existingAttribute, incoming);
+        } else if (!existingAttribute.getTypeName().equals(incoming.getTypeName())) {
+            LOG.info("Attribute type changed for {}.{}: {} -> {}; updating typedef during import",
+                    existing.getName(), incoming.getName(), existingAttribute.getTypeName(), incoming.getTypeName());
 
-                throw new AtlasBaseException(AtlasErrorCode.INVALID_IMPORT_ATTRIBUTE_TYPE_CHANGED, existing.getName(), existingAttribute.getName(), existingAttribute.getTypeName(), incoming.getTypeName());
-            }
+            difference.add(incoming);
         }
     }
 

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java

@@ -177,7 +177,18 @@ public static void updateVertexPreUpdate(AtlasStructDef structDef, AtlasStructTy
                 AtlasAttributeDef existingAttribute = currentStructDef.getAttribute(attributeDef.getName());
 
                 if (null != existingAttribute && !attributeDef.getTypeName().equals(existingAttribute.getTypeName())) {
-                    throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Data type update for attribute is not supported");
+                    if (!RequestContext.get().isImportInProgress()) {
+                        throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Data type update for attribute is not supported");
+                    }
+
+                    LOG.info("Updating attribute type during import: {}.{} from {} to {}",
+                            structDef.getName(), attributeDef.getName(), existingAttribute.getTypeName(), attributeDef.getTypeName());
+
+                    String propertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, attributeDef.getName());
+
+                    AtlasGraphUtilsV2.setProperty(vertex, propertyKey, toJsonFromAttributeDef(attributeDef));
+
+                    continue;
                 }
 
                 String propertyKey = AtlasGraphUtilsV2.getTypeDefPropertyKey(structDef, attributeDef.getName());
@@ -237,16 +248,24 @@ public static AtlasStructDef toStructDef(AtlasVertex vertex, AtlasStructDef stru
 
     @VisibleForTesting
     public static String toJsonFromAttribute(AtlasAttribute attribute) {
-        AtlasAttributeDef   attributeDef = attribute.getAttributeDef();
+        return toJsonFromAttributeDef(attribute.getAttributeDef(), attribute.isOwnedRef(), attribute.getInverseRefAttributeName());
+    }
+
+    @VisibleForTesting
+    public static String toJsonFromAttributeDef(AtlasAttributeDef attributeDef) {
+        return toJsonFromAttributeDef(attributeDef, false, null);
+    }
+
+    private static String toJsonFromAttributeDef(AtlasAttributeDef attributeDef, boolean isComposite, String reverseAttributeName) {
         Map<String, Object> attribInfo   = new HashMap<>();
 
         attribInfo.put("name", attributeDef.getName());
         attribInfo.put("dataType", attributeDef.getTypeName());
         attribInfo.put("isUnique", attributeDef.getIsUnique());
         attribInfo.put("isIndexable", attributeDef.getIsIndexable());
         attribInfo.put("includeInNotification", attributeDef.getIncludeInNotification());
-        attribInfo.put("isComposite", attribute.isOwnedRef());
-        attribInfo.put("reverseAttributeName", attribute.getInverseRefAttributeName());
+        attribInfo.put("isComposite", isComposite);
+        attribInfo.put("reverseAttributeName", reverseAttributeName);
         attribInfo.put("defaultValue", attributeDef.getDefaultValue());
         attribInfo.put("description", attributeDef.getDescription());
         attribInfo.put("searchWeight", attributeDef.getSearchWeight());

repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java

@@ -364,21 +364,18 @@ public void importGlossary(InputStream inputStream) throws IOException, AtlasBas
         assertEntityCount("AtlasGlossaryTerm", "105533b6-c125-4a87-bed5-cdf67fb68c39", 1);
     }
 
-    @Test(dataProvider = "hdfs_path1", expectedExceptions = AtlasBaseException.class)
+    @Test(dataProvider = "hdfs_path1")
     public void importHdfs_path1(InputStream inputStream) throws IOException, AtlasBaseException {
         loadBaseModel();
         loadFsModel();
         loadModelFromResourcesJson("tag1.json", typeDefStore, typeRegistry);
 
-        try {
-            runImportWithNoParameters(importService, inputStream);
-        } catch (AtlasBaseException e) {
-            assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INVALID_IMPORT_ATTRIBUTE_TYPE_CHANGED);
-            AtlasClassificationType tag1 = typeRegistry.getClassificationTypeByName("tag1");
-            assertNotNull(tag1);
-            assertEquals(tag1.getAllAttributes().size(), 2);
-            throw e;
-        }
+        runImportWithNoParameters(importService, inputStream);
+
+        AtlasClassificationType tag1 = typeRegistry.getClassificationTypeByName("tag1");
+        assertNotNull(tag1);
+        assertEquals(tag1.getAttribute("attrib1").getAttributeDef().getTypeName(), "string");
+        assertNotNull(tag1.getAttribute("attrib2"));
     }
 
     @Test(dataProvider = "zip-direct-3", expectedExceptions = AtlasBaseException.class)

repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java

@@ -88,6 +88,21 @@ public void different_ReturnsDifference() throws Exception {
         assertEquals(actualAttributes, expectedAttributes);
     }
 
+    @Test
+    public void attributeTypeChanged_ReturnsUpdatedAttribute() throws Exception {
+        AtlasEntityDef                         existing           = getAtlasEntityDefWithAttributes("name");
+        AtlasEntityDef                         incoming           = new AtlasEntityDef();
+        AtlasStructDef.AtlasAttributeDef       nameAttr           = new AtlasStructDef.AtlasAttributeDef("name", AtlasBaseTypeDef.ATLAS_TYPE_DOUBLE);
+        List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = new ArrayList<>();
+
+        incoming.addAttribute(nameAttr);
+        expectedAttributes.add(nameAttr);
+
+        List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+
+        assertEquals(actualAttributes, expectedAttributes);
+    }
+
     @Test
     public void differentSubset_ReturnsDifference() throws Exception {
         AtlasEntityDef                         existing         = getAtlasEntityDefWithAttributes("name", "qualifiedName");