Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
FALCON-1978 Fix flaky unite test MetadataMappingServicesTests
1. Ordered the tests so that parallel execution will not cause assert failures
2. There was an issue found in EntityGraphRelationshipBuilder that was causing MetadataMappingServicesTests.testOnClusterEntityChange(), but it seems that has been fixed in the trunk.

Author: Venkatesan Ramachandran <vramachandran@hortonworks.com>

Reviewers: "Balu Vellanki <balu@apache.org>"

Closes #158 from vramachan/FALCON-1978.FixUnitTest
  • Loading branch information
vramachan authored and bvellanki committed May 24, 2016
1 parent e7ab836 commit 3fb6e0e756ca476a0551f232d57c0dca28a024c6
Showing 1 changed file with 146 additions and 128 deletions.
@@ -253,7 +253,139 @@ public void testOnAdd() throws Exception {
verifyEntityGraph(RelationshipType.FEED_ENTITY, "Secure");
}

@Test
@Test (dependsOnMethods = "testOnAdd")
// @Test (dependsOnMethods = "testLineageForRetentionWithNoFeedsEvicted")
public void testOnChange() throws Exception {
// shutdown the graph and resurrect for testing
service.destroy();
service.init();

long beforeVerticesCount = getVerticesCount(service.getGraph());
long beforeEdgesCount = getEdgesCount(service.getGraph());

// cannot modify cluster, adding a new cluster
anotherCluster = addClusterEntity("another-cluster", "east-coast",
"classification=another");
verifyEntityWasAddedToGraph("another-cluster", RelationshipType.CLUSTER_ENTITY);

// +3 = cluster, colo, tag, user (but user falcon-user is already added so ignore from count)
Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 3);
// +3 edges to user, colo and new tag
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 3);
}

@Test(dependsOnMethods = "testOnChange")
public void testOnFeedEntityChange() throws Exception {
Feed oldFeed = inputFeeds.get(0);
Feed newFeed = EntityBuilderTestUtil.buildFeed(oldFeed.getName(), clusterEntity,
"classified-as=Secured,source=data-warehouse", "reporting");
addStorage(newFeed, Storage.TYPE.FILESYSTEM,
"jail://global:00/falcon/impression-feed/20140101");

long beforeVerticesCount = 0;
long beforeEdgesCount = 0;

try {
configStore.initiateUpdate(newFeed);

beforeVerticesCount = getVerticesCount(service.getGraph());
beforeEdgesCount = getEdgesCount(service.getGraph());

// add cluster
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
new org.apache.falcon.entity.v0.feed.Cluster();
feedCluster.setName(anotherCluster.getName());
newFeed.getClusters().getClusters().add(feedCluster);

configStore.update(EntityType.FEED, newFeed);
} finally {
configStore.cleanupUpdateInit();
}

verifyUpdatedEdges(newFeed);
Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 2); //+2 = 2 new tags
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 2); // +2 = 1 new cluster, 1 new tag
}

@Test(dependsOnMethods = "testOnFeedEntityChange")
public void testOnProcessEntityChange() throws Exception {
long beforeVerticesCount = getVerticesCount(service.getGraph());
long beforeEdgesCount = getEdgesCount(service.getGraph());

Process oldProcess = processEntity;
Process newProcess = EntityBuilderTestUtil.buildProcess(oldProcess.getName(), anotherCluster,
null, null);
EntityBuilderTestUtil.addProcessWorkflow(newProcess, GENERATE_WORKFLOW_NAME, "2.0.0");
EntityBuilderTestUtil.addInput(newProcess, inputFeeds.get(0));

try {
configStore.initiateUpdate(newProcess);
configStore.update(EntityType.PROCESS, newProcess);
} finally {
configStore.cleanupUpdateInit();
}

verifyUpdatedEdges(newProcess);
Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 0); // +0, no net new
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount - 6); // -6 = -2 outputs, -1 tag,
// -1 cluster, -2 pipelines
}

@Test(dependsOnMethods = "testOnProcessEntityChange")
public void testAreSame() throws Exception {

Inputs inputs1 = new Inputs();
Inputs inputs2 = new Inputs();
Outputs outputs1 = new Outputs();
Outputs outputs2 = new Outputs();
// return true when both are null
Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));

Input i1 = new Input();
i1.setName("input1");
Input i2 = new Input();
i2.setName("input2");
Output o1 = new Output();
o1.setName("output1");
Output o2 = new Output();
o2.setName("output2");

inputs1.getInputs().add(i1);
Assert.assertFalse(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
outputs1.getOutputs().add(o1);
Assert.assertFalse(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));

inputs2.getInputs().add(i1);
Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
outputs2.getOutputs().add(o1);
Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));
}

@Test(dependsOnMethods = "testAreSame")
public void testOnClusterEntityChange() throws Exception {
long beforeVerticesCount = getVerticesCount(service.getGraph());
long beforeEdgesCount = getEdgesCount(service.getGraph());

Cluster oldCluster = clusterEntity;
Cluster newCluster = EntityBuilderTestUtil.buildCluster(oldCluster.getName(),
"clusterUpdateColo", oldCluster.getTags() + ",clusterUpdateTagKey=clusterUpdateTagVal");

try {
configStore.initiateUpdate(newCluster);
configStore.update(EntityType.CLUSTER, newCluster);
} finally {
configStore.cleanupUpdateInit();
}

Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 2); // +1 new tag +1 new colo
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 1); // +1 new tag edge
Vertex newClusterVertex = getEntityVertex(newCluster.getName(), RelationshipType.CLUSTER_ENTITY);
verifyVertexForEdge(newClusterVertex, Direction.OUT, RelationshipLabel.CLUSTER_COLO.getName(),
"clusterUpdateColo", RelationshipType.COLO.getName());
}

@Test (dependsOnMethods = "testOnClusterEntityChange")
public void testMapLineage() throws Exception {
setup();

@@ -275,7 +407,7 @@ public void testMapLineage() throws Exception {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 40);
}

@Test
@Test (dependsOnMethods = "testMapLineage")
public void testLineageForNoDateInFeedPath() throws Exception {
setupForNoDateInFeedPath();

@@ -304,7 +436,7 @@ public void testLineageForNoDateInFeedPath() throws Exception {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 34);
}

@Test
@Test (dependsOnMethods = "testLineageForNoDateInFeedPath")
public void testLineageForReplication() throws Exception {
setupForLineageReplication();

@@ -344,7 +476,7 @@ public void testLineageForReplication() throws Exception {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 1);
}

@Test
@Test (dependsOnMethods = "testLineageForReplication")
public void testLineageForReplicationForNonGeneratedInstances() throws Exception {
cleanUp();
service.init();
@@ -374,7 +506,7 @@ public void testLineageForReplicationForNonGeneratedInstances() throws Exception
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 6);
}

@Test
@Test (dependsOnMethods = "testLineageForReplicationForNonGeneratedInstances")
public void testLineageForRetention() throws Exception {
setupForLineageEviction();
// Get the before vertices and edges
@@ -409,7 +541,7 @@ public void testLineageForRetention() throws Exception {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 2);
}

@Test
@Test (dependsOnMethods = "testLineageForRetention")
public void testLineageForRetentionWithNoFeedsEvicted() throws Exception {
cleanUp();
service.init();
@@ -430,59 +562,20 @@ public void testLineageForRetentionWithNoFeedsEvicted() throws Exception {
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount);
}

@Test (dependsOnMethods = "testOnAdd")
public void testOnChange() throws Exception {
// shutdown the graph and resurrect for testing
service.destroy();
service.init();

long beforeVerticesCount = getVerticesCount(service.getGraph());
long beforeEdgesCount = getEdgesCount(service.getGraph());

// cannot modify cluster, adding a new cluster
anotherCluster = addClusterEntity("another-cluster", "east-coast",
"classification=another");
verifyEntityWasAddedToGraph("another-cluster", RelationshipType.CLUSTER_ENTITY);
private String printProps(Vertex ignored) {

Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 3); // +3 = cluster, colo, tag
// +3 edges to user, colo and new tag
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 3);
StringBuilder sb = new StringBuilder();
for(String p : ignored.getPropertyKeys()) {
sb.append(p).append("->").append(ignored.getProperty(p) + ";");
}
return sb.toString();
}

@Test(dependsOnMethods = "testOnChange")
public void testOnFeedEntityChange() throws Exception {
Feed oldFeed = inputFeeds.get(0);
Feed newFeed = EntityBuilderTestUtil.buildFeed(oldFeed.getName(), clusterEntity,
"classified-as=Secured,source=data-warehouse", "reporting");
addStorage(newFeed, Storage.TYPE.FILESYSTEM,
"jail://global:00/falcon/impression-feed/20140101");

long beforeVerticesCount = 0;
long beforeEdgesCount = 0;

try {
configStore.initiateUpdate(newFeed);

beforeVerticesCount = getVerticesCount(service.getGraph());
beforeEdgesCount = getEdgesCount(service.getGraph());

// add cluster
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
new org.apache.falcon.entity.v0.feed.Cluster();
feedCluster.setName(anotherCluster.getName());
newFeed.getClusters().getClusters().add(feedCluster);

configStore.update(EntityType.FEED, newFeed);
} finally {
configStore.cleanupUpdateInit();
}

verifyUpdatedEdges(newFeed);
Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 2); //+2 = 2 new tags
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 2); // +2 = 1 new cluster, 1 new tag
}

@Test
@Test (dependsOnMethods = "testLineageForRetentionWithNoFeedsEvicted")
public void testLineageForTransactionFailure() throws Exception {
cleanUp();
service.init();
@@ -531,62 +624,8 @@ private void verifyUpdatedEdges(Feed newFeed) {
"Actual does not contain expected: " + actual);
}

@Test(dependsOnMethods = "testOnFeedEntityChange")
public void testOnProcessEntityChange() throws Exception {
long beforeVerticesCount = getVerticesCount(service.getGraph());
long beforeEdgesCount = getEdgesCount(service.getGraph());

Process oldProcess = processEntity;
Process newProcess = EntityBuilderTestUtil.buildProcess(oldProcess.getName(), anotherCluster,
null, null);
EntityBuilderTestUtil.addProcessWorkflow(newProcess, GENERATE_WORKFLOW_NAME, "2.0.0");
EntityBuilderTestUtil.addInput(newProcess, inputFeeds.get(0));

try {
configStore.initiateUpdate(newProcess);
configStore.update(EntityType.PROCESS, newProcess);
} finally {
configStore.cleanupUpdateInit();
}

verifyUpdatedEdges(newProcess);
Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 0); // +0, no net new
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount - 6); // -6 = -2 outputs, -1 tag,
// -1 cluster, -2 pipelines
}

@Test(dependsOnMethods = "testOnProcessEntityChange")
public void testAreSame() throws Exception {

Inputs inputs1 = new Inputs();
Inputs inputs2 = new Inputs();
Outputs outputs1 = new Outputs();
Outputs outputs2 = new Outputs();
// return true when both are null
Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));

Input i1 = new Input();
i1.setName("input1");
Input i2 = new Input();
i2.setName("input2");
Output o1 = new Output();
o1.setName("output1");
Output o2 = new Output();
o2.setName("output2");

inputs1.getInputs().add(i1);
Assert.assertFalse(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
outputs1.getOutputs().add(o1);
Assert.assertFalse(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));

inputs2.getInputs().add(i1);
Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(inputs1, inputs2));
outputs2.getOutputs().add(o1);
Assert.assertTrue(EntityRelationshipGraphBuilder.areSame(outputs1, outputs2));
}

@Test
@Test (dependsOnMethods = "testLineageForRetentionWithNoFeedsEvicted")
public void testLineageForJobCounter() throws Exception {
setupForJobCounters();
WorkflowExecutionContext context = WorkflowExecutionContext.create(getTestMessageArgs(
@@ -606,28 +645,7 @@ public void testLineageForJobCounter() throws Exception {
verifyLineageGraphForJobCounters(context);
}

@Test(dependsOnMethods = "testOnFeedEntityChange")
public void testOnClusterEntityChange() throws Exception {
long beforeVerticesCount = getVerticesCount(service.getGraph());
long beforeEdgesCount = getEdgesCount(service.getGraph());

Cluster oldCluster = clusterEntity;
Cluster newCluster = EntityBuilderTestUtil.buildCluster(oldCluster.getName(),
"clusterUpdateColo", oldCluster.getTags() + ",clusterUpdateTagKey=clusterUpdateTagVal");

try {
configStore.initiateUpdate(newCluster);
configStore.update(EntityType.CLUSTER, newCluster);
} finally {
configStore.cleanupUpdateInit();
}

Assert.assertEquals(getVerticesCount(service.getGraph()), beforeVerticesCount + 2); // +1 new tag +1 new colo
Assert.assertEquals(getEdgesCount(service.getGraph()), beforeEdgesCount + 1); // +1 new tag edge
Vertex newClusterVertex = getEntityVertex(newCluster.getName(), RelationshipType.CLUSTER_ENTITY);
verifyVertexForEdge(newClusterVertex, Direction.OUT, RelationshipLabel.CLUSTER_COLO.getName(),
"clusterUpdateColo", RelationshipType.COLO.getName());
}

private void verifyUpdatedEdges(Process newProcess) {
Vertex processVertex = getEntityVertex(newProcess.getName(), RelationshipType.PROCESS_ENTITY);

0 comments on commit 3fb6e0e

Please sign in to comment.