FeedEntityParserTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.ivory.entity.parser;

/**
 * Test Cases for ProcessEntityParser
 */
import static org.testng.AssertJUnit.assertEquals;

import java.io.IOException;
import java.io.StringWriter;

import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
import javax.xml.bind.Unmarshaller;

import org.apache.ivory.IvoryException;
import org.apache.ivory.entity.AbstractTestBase;
import org.apache.ivory.entity.FeedHelper;
import org.apache.ivory.entity.store.ConfigurationStore;
import org.apache.ivory.entity.v0.EntityType;
import org.apache.ivory.entity.v0.Frequency;
import org.apache.ivory.entity.v0.SchemaHelper;
import org.apache.ivory.entity.v0.cluster.Cluster;
import org.apache.ivory.entity.v0.feed.ActionType;
import org.apache.ivory.entity.v0.feed.ClusterType;
import org.apache.ivory.entity.v0.feed.Feed;
import org.apache.ivory.entity.v0.feed.Location;
import org.apache.ivory.entity.v0.feed.LocationType;
import org.apache.ivory.entity.v0.feed.Locations;
import org.apache.ivory.entity.v0.feed.Validity;
import org.apache.ivory.group.FeedGroup;
import org.apache.ivory.group.FeedGroupMapTest;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;

public class FeedEntityParserTest extends AbstractTestBase {

	private final FeedEntityParser parser = (FeedEntityParser) EntityParserFactory
			.getParser(EntityType.FEED);

	private Feed modifiableFeed;

	@BeforeMethod
	public void setUp() throws Exception {
	    cleanupStore();
	    ConfigurationStore store = ConfigurationStore.get();
	    
		Unmarshaller unmarshaller = EntityType.CLUSTER.getUnmarshaller();
		Cluster cluster = (Cluster) unmarshaller.unmarshal(this.getClass()
				.getResourceAsStream(CLUSTER_XML));
		cluster.setName("testCluster");
		store.publish(EntityType.CLUSTER, cluster);

		cluster = (Cluster) unmarshaller.unmarshal(this.getClass()
				.getResourceAsStream(CLUSTER_XML));
		cluster.setName("backupCluster");
		store.publish(EntityType.CLUSTER, cluster);

		modifiableFeed = (Feed) parser.parseAndValidate(this.getClass()
				.getResourceAsStream(FEED_XML));
	}

	@Test(expectedExceptions = ValidationException.class)
	public void testValidations() throws Exception {
		ConfigurationStore.get().remove(EntityType.CLUSTER, "backupCluster");
		parser.parseAndValidate(this.getClass().getResourceAsStream(FEED_XML));
	}

	@Test
	public void testParse() throws IOException, IvoryException, JAXBException {

		Feed feed = parser.parseAndValidate(this.getClass()
				.getResourceAsStream(FEED_XML));

		Assert.assertNotNull(feed);
		assertEquals(feed.getName(), "clicks");
		assertEquals(feed.getDescription(), "clicks log");
		assertEquals(feed.getFrequency().toString(), "hours(1)");
		assertEquals(feed.getGroups(), "online,bi");

		assertEquals(feed.getClusters().getClusters().get(0).getName(),
				"testCluster");
		assertEquals(feed.getClusters().getClusters().get(0).getType(),
				ClusterType.SOURCE);
		assertEquals(SchemaHelper.formatDateUTC(feed.getClusters().getClusters().get(0).getValidity()
				.getStart()), "2011-11-01T00:00Z");
		assertEquals(SchemaHelper.formatDateUTC(feed.getClusters().getClusters().get(0).getValidity()
				.getEnd()), "2011-12-31T00:00Z");
		assertEquals(feed.getTimezone().getID(), "UTC");
		assertEquals(feed.getClusters().getClusters().get(0).getRetention()
				.getAction(), ActionType.DELETE);
		assertEquals(feed.getClusters().getClusters().get(0).getRetention()
				.getLimit().toString(), "hours(48)");

		assertEquals(feed.getClusters().getClusters().get(1).getName(),
				"backupCluster");
		assertEquals(feed.getClusters().getClusters().get(1).getType(),
				ClusterType.TARGET);
		assertEquals(SchemaHelper.formatDateUTC(feed.getClusters().getClusters().get(1).getValidity()
				.getStart()), "2011-11-01T00:00Z");
		assertEquals(SchemaHelper.formatDateUTC(feed.getClusters().getClusters().get(1).getValidity()
				.getEnd()), "2011-12-31T00:00Z");
		assertEquals(feed.getClusters().getClusters().get(1).getRetention()
				.getAction(), ActionType.ARCHIVE);
		assertEquals(feed.getClusters().getClusters().get(1).getRetention()
				.getLimit().toString(), "hours(6)");

		assertEquals(FeedHelper.getLocation(feed, LocationType.DATA).getPath(),
				"/projects/ivory/clicks");
		assertEquals(FeedHelper.getLocation(feed, LocationType.META).getPath(),
				"/projects/ivory/clicksMetaData");
		assertEquals(FeedHelper.getLocation(feed, LocationType.STATS).getPath(),
				"/projects/ivory/clicksStats");

		assertEquals(feed.getACL().getGroup(), "group");
		assertEquals(feed.getACL().getOwner(), "testuser");
		assertEquals(feed.getACL().getPermission(), "0x755");

		assertEquals(feed.getSchema().getLocation(), "/schema/clicks");
		assertEquals(feed.getSchema().getProvider(), "protobuf");

		StringWriter stringWriter = new StringWriter();
		Marshaller marshaller = EntityType.FEED.getMarshaller();
		marshaller.marshal(feed, stringWriter);
		System.out.println(stringWriter.toString());
	}

	@Test(expectedExceptions = ValidationException.class)
	public void applyValidationInvalidFeed() throws Exception {
		Feed feed = (Feed) parser
				.parseAndValidate(ProcessEntityParserTest.class
						.getResourceAsStream(FEED_XML));
		feed.getClusters().getClusters().get(0).setName("invalid cluster");
		parser.validate(feed);
	}

	
	@Test
	public void testPartitionExpression() throws IvoryException {
        Feed feed = (Feed) parser.parseAndValidate(ProcessEntityParserTest.class
                .getResourceAsStream(FEED_XML));
        
        //When there are more than 1 src clusters, there should be partition expression
        org.apache.ivory.entity.v0.feed.Cluster newCluster = new org.apache.ivory.entity.v0.feed.Cluster();
        newCluster.setName("newCluster");
        newCluster.setType(ClusterType.SOURCE);
        newCluster.setPartition("${cluster.colo}");
        feed.getClusters().getClusters().add(newCluster);
        try {
            parser.validate(feed);
            Assert.fail("Expected ValidationException");
        } catch(ValidationException e) { }
        
        //When there are more than 1 src clusters, the partition expression should contain cluster variable
        feed.getClusters().getClusters().get(0).setPartition("*");
        try {
            parser.validate(feed);
            Assert.fail("Expected ValidationException");
        } catch(ValidationException e) { }
        
        //When there are more than 1 target cluster, there should be partition expre
        newCluster.setType(ClusterType.TARGET);
        try {
            parser.validate(feed);
            Assert.fail("Expected ValidationException");
        } catch(ValidationException e) { }
        
        //When there are more than 1 target clusters, the partition expression should contain cluster variable
        feed.getClusters().getClusters().get(1).setPartition("*");
        try {
            parser.validate(feed);
            Assert.fail("Expected ValidationException");
        } catch(ValidationException e) { }
        
        //Number of parts in partition expression < number of partitions defined for feed
        feed.getClusters().getClusters().get(1).setPartition("*/*");
        try {
            parser.validate(feed);
            Assert.fail("Expected ValidationException");
        } catch(ValidationException e) { }

        feed.getClusters().getClusters().get(0).setPartition(null);
        feed.getClusters().getClusters().get(1).setPartition(null);
        feed.getClusters().getClusters().remove(2);
        feed.setPartitions(null);
        parser.validate(feed);
	}

	@Test
	public void testInvalidClusterValidityTime() {
		Validity validity = modifiableFeed.getClusters().getClusters().get(0)
				.getValidity();
		try {
			validity.setStart(SchemaHelper.parseDateUTC("2007-02-29T00:00Z"));
			modifiableFeed.getClusters().getClusters().get(0)
					.setValidity(validity);
			parser.parseAndValidate(marshallEntity(modifiableFeed));
			Assert.fail("Cluster validity failed");
		} catch (Exception e) {
			System.out.println(e.getMessage());
			validity.setStart(SchemaHelper.parseDateUTC("2011-11-01T00:00Z"));
			modifiableFeed.getClusters().getClusters().get(0)
					.setValidity(validity);
		}

		try {
			validity.setEnd(SchemaHelper.parseDateUTC("2010-04-31T00:00Z"));
			modifiableFeed.getClusters().getClusters().get(0)
					.setValidity(validity);
			parser.parseAndValidate(marshallEntity(modifiableFeed));
			Assert.fail("Cluster validity failed");
		} catch (Exception e) {
			System.out.println(e.getMessage());
			validity.setEnd(SchemaHelper.parseDateUTC("2011-12-31T00:00Z"));
			modifiableFeed.getClusters().getClusters().get(0)
					.setValidity(validity);
		}
	}

	@Test(expectedExceptions = ValidationException.class)
	public void testInvalidProcessValidity() throws Exception {
		Feed feed = parser.parseAndValidate((FeedEntityParserTest.class
				.getResourceAsStream(FEED_XML)));
		feed.getClusters().getClusters().get(0).getValidity()
				.setStart(SchemaHelper.parseDateUTC("2012-11-01T00:00Z"));
		parser.validate(feed);
	}

	@Test
	public void testValidFeedGroup() throws IvoryException, JAXBException {
		Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				(FeedEntityParserTest.class.getResourceAsStream(FEED_XML)));
		feed1.setName("f1" + System.currentTimeMillis());
		feed1.setGroups("group1,group2,group3");
		feed1.setLocations(new Locations());
		Location location = new Location();
		location.setPath("/projects/bi/rmc/daily/ad/${YEAR}/fraud/${MONTH}-${DAY}/ad");
		location.setType(LocationType.DATA);
		feed1.getLocations().getLocations().add(location);
		feed1.getClusters().getClusters().get(0).getLocations().getLocations().set(0, location);
		parser.parseAndValidate(feed1.toString());
		ConfigurationStore.get().publish(EntityType.FEED, feed1);

		Feed feed2 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				(FeedEntityParserTest.class.getResourceAsStream(FEED_XML)));
		feed2.setName("f2" + System.currentTimeMillis());
		feed2.setGroups("group1,group2,group5");
		feed2.setLocations(new Locations());
		Location location2 = new Location();
		location2
				.setPath("/projects/bi/rmc/daily/ad/${YEAR}/fraud/${MONTH}-${DAY}/ad");
		location2.setType(LocationType.DATA);
		feed2.getLocations().getLocations().add(location2);
		feed2.getClusters().getClusters().get(0).getLocations().getLocations().set(0, location);
		parser.parseAndValidate(feed2.toString());
	}
	
	@Test(expectedExceptions = ValidationException.class)
	public void testInvalidFeedClusterDataLocation() throws JAXBException, IvoryException{
		Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				(FeedEntityParserTest.class.getResourceAsStream(FEED_XML)));
		feed1.setName("f1" + System.currentTimeMillis());
		feed1.setGroups("group1,group2,group3");
		feed1.setLocations(new Locations());
		Location location = new Location();
		location.setPath("/projects/bi/rmc/daily/ad/${YEAR}/fraud/${MONTH}-${DAY}/ad");
		location.setType(LocationType.DATA);
		feed1.getLocations().getLocations().add(location);
		parser.parseAndValidate(feed1.toString());
	}

	@Test(expectedExceptions = ValidationException.class)
	public void testInvalidFeedGroup() throws IvoryException, JAXBException {
		Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				(FeedEntityParserTest.class.getResourceAsStream(FEED_XML)));
		feed1.setName("f1" + System.currentTimeMillis());
		feed1.setGroups("group1,group2,group3");
		feed1.setLocations(new Locations());
		Location location = new Location();
		location.setPath("/projects/bi/rmc/daily/ad/${YEAR}/fraud/${MONTH}-${DAY}/ad");
		location.setType(LocationType.DATA);
		feed1.getLocations().getLocations().add(location);
		parser.parseAndValidate(feed1.toString());
		
		feed1.getClusters().getClusters().get(0).getLocations().getLocations().set(0,location);
		ConfigurationStore.get().publish(EntityType.FEED, feed1);

		Feed feed2 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				(FeedEntityParserTest.class.getResourceAsStream(FEED_XML)));
		feed2.setName("f2" + System.currentTimeMillis());
		feed2.setGroups("group1,group2,group5");
		feed2.setLocations(new Locations());
		Location location2 = new Location();
		location2
				.setPath("/projects/bi/rmc/daily/ad/${YEAR}/fraud/${MONTH}/${HOUR}/ad");
		location2.setType(LocationType.DATA);
		feed2.getLocations().getLocations().add(location2);
		feed2.getClusters().getClusters().get(0).getLocations().getLocations().set(0,location);
		parser.parseAndValidate(feed2.toString());
	}

	@Test
	public void testValidGroupNames() throws IvoryException, JAXBException {
		Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				FeedGroupMapTest.class
						.getResourceAsStream("/config/feed/feed-0.1.xml"));
		feed1.setName("f1" + System.currentTimeMillis());
		feed1.setGroups("group7,group8");
		parser.parseAndValidate(feed1.toString());

		feed1.setGroups("group7");
		parser.parseAndValidate(feed1.toString());

		feed1.setGroups(null);
		parser.parseAndValidate(feed1.toString());
		ConfigurationStore.get().publish(EntityType.FEED, feed1);
	}

	@Test
	public void testInvalidGroupNames() throws IvoryException, JAXBException {
		Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				FeedGroupMapTest.class
						.getResourceAsStream("/config/feed/feed-0.1.xml"));
		feed1.setName("f1" + System.currentTimeMillis());

		try {
			feed1.setGroups("commaend,");
			parser.parseAndValidate(feed1.toString());
			Assert.fail("Expected exception");
		} catch (IvoryException e) {

		}
		try {
			feed1.setGroups("group8,   group9");
			parser.parseAndValidate(feed1.toString());
			Assert.fail("Expected exception");
		} catch (IvoryException e) {

		}
		try {
			feed1.setGroups("space in group,group9");
			parser.parseAndValidate(feed1.toString());
			Assert.fail("Expected exception");
		} catch (IvoryException e) {

		}
	}
	
	@Test
	public void testClusterPartitionExp() throws IvoryException {
		Cluster cluster = ConfigurationStore.get().get(EntityType.CLUSTER,
				"testCluster");
		Assert.assertEquals(FeedHelper.evaluateClusterExp(cluster,
				"/*/${cluster.colo}"), "/*/" + cluster.getColo());
		Assert.assertEquals(FeedHelper.evaluateClusterExp(cluster,
				"/*/${cluster.name}/Local"), "/*/" + cluster.getName()+"/Local");
		Assert.assertEquals(FeedHelper.evaluateClusterExp(cluster,
				"/*/${cluster.field1}/Local"), "/*/value1/Local");
	}
	
	@Test(expectedExceptions=IvoryException.class)
	public void testInvalidFeedName() throws JAXBException, IvoryException  {
		Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				FeedGroupMapTest.class
						.getResourceAsStream("/config/feed/feed-0.1.xml"));
		feed1.setName("Feed_name");
		parser.parseAndValidate(feed1.toString());
	}
	
	@Test(expectedExceptions=IvoryException.class)
	public void testInvalidFeedGroupName() throws JAXBException, IvoryException  {
		Feed feed1 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				FeedGroupMapTest.class
						.getResourceAsStream("/config/feed/feed-0.1.xml"));
		feed1.setName("feed1");
		feed1.getLocations().getLocations().get(0)
				.setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
		feed1.getClusters().getClusters().get(0).getLocations().getLocations()
				.get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
		ConfigurationStore.get().publish(EntityType.FEED, feed1);
		
		Feed feed2 = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				FeedGroupMapTest.class
						.getResourceAsStream("/config/feed/feed-0.1.xml"));
		feed2.setName("feed2");
		feed2.getLocations().getLocations().get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
		feed2.getClusters().getClusters().get(0).getLocations().getLocations()
		.get(0).setPath("/data/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
		feed2.setFrequency(new Frequency("hours(1)"));
		try{
			parser.parseAndValidate(feed2.toString());
		}catch(IvoryException e){
			e.printStackTrace();
			Assert.fail("Not expecting exception for same frequency");
		}
		feed2.setFrequency(new Frequency("hours(2)"));
		//expecting exception
		parser.parseAndValidate(feed2.toString());
	}
	
	@Test
	public void testNullFeedLateArrival() throws JAXBException, IvoryException  {
		Feed feed = (Feed) EntityType.FEED.getUnmarshaller().unmarshal(
				FeedGroupMapTest.class
						.getResourceAsStream("/config/feed/feed-0.1.xml"));
		
		feed.setLateArrival(null);
		parser.parseAndValidate(feed.toString());
		
	}

}