Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

dataIn to construct paths even for non-input datasets - for optional …

…inputs
  • Loading branch information...
commit e9d3db95b3116dfae3da24309c41d030e97253d6 1 parent bb8fdd1
Shwetha GS authored
View
58 src/main/java/org/apache/oozie/extensions/OozieELExtensions.java
@@ -21,7 +21,9 @@
import java.util.Calendar;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.command.coord.CoordCommandUtils;
+import org.apache.oozie.coord.CoordELEvaluator;
import org.apache.oozie.coord.CoordELFunctions;
import org.apache.oozie.coord.SyncCoordAction;
import org.apache.oozie.coord.SyncCoordDataset;
@@ -29,6 +31,9 @@
import org.apache.oozie.util.ELEvaluator;
import org.apache.oozie.util.ParamChecker;
import org.apache.oozie.util.XLog;
+import org.jdom.Attribute;
+import org.jdom.Element;
+import org.jdom.Text;
public class OozieELExtensions {
@@ -39,21 +44,30 @@
public static final String COORD_CURRENT = "coord:current";
public static String ph1_dataIn_echo(String dataInName, String part) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- String val = (String) eval.getVariable("oozie.dataname." + dataInName);
- if (val == null || !val.equals("data-in")) {
- XLog.getLog(CoordELFunctions.class).error("data_in_name " + dataInName + " is not valid");
- throw new RuntimeException("data_in_name " + dataInName + " is not valid");
- }
return "dataIn('" + dataInName + "', '" + part + "')";
}
public static String ph3_dataIn(String dataInName, String part) {
ELEvaluator eval = ELEvaluator.getCurrent();
String uristr = (String) eval.getVariable(".datain." + dataInName);
- Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
- if (unresolved != null && unresolved) {
- throw new RuntimeException("There are unresolved instances in " + uristr);
+ if(uristr == null) { //optional input
+ Element dsEle = getDSElement(eval, dataInName);
+ Configuration conf = new Configuration();
+ SyncCoordAction appInst = (SyncCoordAction) eval.getVariable(CoordELFunctions.COORD_ACTION);
+ try {
+ ELEvaluator instEval = CoordELEvaluator.createInstancesELEvaluator(dsEle, appInst, conf);
+ StringBuilder instances = new StringBuilder();
+ CoordCommandUtils.resolveInstanceRange(dsEle, instances , appInst, conf, instEval);
+ uristr = CoordCommandUtils.createEarlyURIs(dsEle, instances.toString(), new StringBuilder(), new StringBuilder());
+ uristr = uristr.replace(CoordELFunctions.INSTANCE_SEPARATOR, ",");
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to resolve instance range for " + dataInName, e);
+ }
+ } else {
+ Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
+ if (unresolved != null && unresolved) {
+ throw new RuntimeException("There are unresolved instances in " + uristr);
+ }
}
if (StringUtils.isNotEmpty(uristr) && StringUtils.isNotEmpty(part) && !part.equals("null")) {
@@ -71,6 +85,32 @@ public static String ph3_dataIn(String dataInName, String part) {
return uristr;
}
+ private static Element getDSElement(ELEvaluator eval, String dataInName) {
+ Element ele = new Element("datain");
+ Element dsEle = new Element("dataset");
+ ele.getChildren().add(dsEle);
+
+ String[] attrs = {"initial-instance", "frequency", "freq_timeunit", "timezone", "end_of_duration"};
+ for(String attr:attrs)
+ dsEle.getAttributes().add(new Attribute(attr, (String) eval.getVariable(dataInName + "." + attr)));
+
+ String[] children = {"done-flag", "uri-template"};
+ for(String child:children) {
+ Element childEle = new Element(child);
+ childEle.setContent(new Text(((String) eval.getVariable(dataInName + "." + child)).replace('%', '$')));
+ dsEle.getChildren().add(childEle);
+ }
+
+ String[] eleChildren = {"start-instance", "end-instance"};
+ for(String child:eleChildren) {
+ Element childEle = new Element(child);
+ childEle.setContent(new Text("${" + ((String) eval.getVariable(dataInName + "." + child)) + "}"));
+ ele.getChildren().add(childEle);
+ }
+
+ return ele;
+ }
+
public static String ph1_now_echo(int hr, int min) {
ELEvaluator eval = ELEvaluator.getCurrent();
eval.setVariable(".wrap", "true");
View
333 src/main/java/org/apache/oozie/extensions/OozieELExtensionsOld.java
@@ -1,333 +0,0 @@
-package org.apache.oozie.extensions;
-
-import java.util.Calendar;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.oozie.command.coord.CoordCommandUtils;
-import org.apache.oozie.coord.CoordELFunctions;
-import org.apache.oozie.util.ELEvaluator;
-import org.apache.oozie.util.XLog;
-
-public class OozieELExtensionsOld {
-
- private enum TruncateBoundary {
- NONE, DAY, MONTH, QUARTER, YEAR;
- }
-
- private static final String PREFIX = "elext:";
- public static final String COORD_CURRENT = "coord:current";
-
- public static String ph1_dataIn_echo(String dataInName, String part) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- String val = (String) eval.getVariable("oozie.dataname." + dataInName);
- if (val == null || !val.equals("data-in")) {
- XLog.getLog(CoordELFunctions.class).error("data_in_name " + dataInName + " is not valid");
- throw new RuntimeException("data_in_name " + dataInName + " is not valid");
- }
- return PREFIX + "dataIn('" + dataInName + "', '" + part + "')";
- }
-
-
- public static String ph3_dataIn(String dataInName, String part) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- String uristr = (String) eval.getVariable(".datain." + dataInName);
- Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
- if (unresolved != null && unresolved) {
- throw new RuntimeException("There are unresolved instances in " + uristr);
- }
-
- if(StringUtils.isNotEmpty(uristr) && StringUtils.isNotEmpty(part)) {
- String[] uris = uristr.split(",");
- StringBuilder mappedUris = new StringBuilder();
- for(String uri:uris) {
- if(uri.trim().length() == 0)
- continue;
- if(mappedUris.length() > 0)
- mappedUris.append(",");
- mappedUris.append(uri).append("/").append(part);
- }
- return mappedUris.toString();
- }
- return uristr;
- }
-
- public static String ph1_now_echo(int hr, int min) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "now("+ hr + "," + min + ")"; // Unresolved
- }
-
- public static String ph1_today_echo(int hr, int min) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "today(" + hr + ", " + min + ")"; // Unresolved
- }
-
- public static String ph1_todayWithOffset_echo(int hr, int min, int offsetMin) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "todayWithOffset(" + hr + ", " + min + ", " + offsetMin + ")"; // Unresolved
- }
-
- public static String ph1_yesterday_echo(int hr, int min) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "yesterday(" + hr + ", " + min + ")"; // Unresolved
- }
-
- public static String ph1_yesterdayWithOffset_echo(int hr, int min, int offsetMin) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "yesterdayWithOffset(" + hr + ", " + min + ", " + offsetMin + ")"; // Unresolved
- }
-
- public static String ph1_currentMonth_echo(int day, int hr, int min) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "currentMonth(" + day + ", " + hr + ", " + min + ")"; // Unresolved
- }
-
- public static String ph1_currentMonthWithOffset_echo(int day, int hr, int min, int offsetMin) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "currentMonth(" + day + ", " + hr + ", " + min + ", " + offsetMin + ")"; // Unresolved
- }
-
- public static String ph1_lastMonth_echo(int day, int hr, int min) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "lastMonth(" + day + ", " + hr + ", " + min + ")"; // Unresolved
- }
-
- public static String ph1_lastMonthWithOffset_echo(int day, int hr, int min, int offsetMin) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "lastMonth(" + day + ", " + hr + ", " + min + ", " + offsetMin + ")"; // Unresolved
- }
-
- public static String ph1_currentYear_echo(int month, int day, int hr, int min) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "currentYear(" + month + ", " + day + ", " + hr + ", " + min + ")"; // Unresolved
- }
-
- public static String ph1_currentYearWithOffset_echo(int month, int day, int hr,
- int min, int offsetMin) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "currentYear(" + month + ", " + day + ", " + hr + ", " +
- min + ", " + offsetMin + ")"; // Unresolved
- }
-
- public static String ph1_lastYear_echo(int month, int day, int hr, int min) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "lastYear(" + month + ", " + day + ", " + hr + ", " + min + ")"; // Unresolved
- }
-
- public static String ph1_lastYearWithOffset_echo(int month, int day, int hr,
- int min, int offsetMin) {
- ELEvaluator eval = ELEvaluator.getCurrent();
- eval.setVariable(".wrap", "true");
- return PREFIX + "lastYear(" + month + ", " + day + ", " + hr + ", " +
- min + ", " + offsetMin + ")"; // Unresolved
- }
-
- public static String ph2_now_inst(int hr, int min) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.NONE, 0, 0, 0, hr, min, 0);
- }
-
- public static String ph2_today_inst(int hr, int min) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.DAY, 0, 0, 0, hr, min, 0);
- }
-
- public static String ph2_todayWithOffset_inst(int hr, int min, int offsetMin) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.DAY, 0, 0, 0, hr, min, offsetMin);
- }
-
- public static String ph2_yesterday_inst(int hr, int min) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.DAY, 0, 0, -1, hr, min, 0);
- }
-
- public static String ph2_yesterdayWithOffset_inst(int hr, int min, int offsetMin) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.DAY, 0, 0, -1, hr, min, offsetMin);
- }
-
- public static String ph2_currentMonth_inst(int day, int hr, int min) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.MONTH, 0, 0, day, hr, min, 0);
- }
-
- public static String ph2_currentMonthWithOffset_inst(int day, int hr, int min, int offsetMin) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.MONTH, 0, 0, day, hr, min, offsetMin);
- }
-
- public static String ph2_lastMonth_inst(int day, int hr, int min) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.MONTH, 0, -1, day, hr, min, 0);
- }
-
- public static String ph2_lastMonthWithOffset_inst(int day, int hr, int min, int offsetMin) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.MONTH, 0, -1, day, hr, min, offsetMin);
- }
-
- public static String ph2_currentYear_inst(int month, int day, int hr, int min) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.YEAR, 0, month, day, hr, min, 0);
- }
-
- public static String ph2_currentYearWithOffset_inst(int month, int day,
- int hr, int min, int offsetMin) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.YEAR, 0, month, day, hr, min, offsetMin);
- }
-
- public static String ph2_lastYear_inst(int month, int day, int hr, int min) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.YEAR, -1, month, day, hr, min, 0);
- }
-
- public static String ph2_lastYearWithOffset_inst(int month, int day,
- int hr, int min, int offsetMin) throws Exception {
- return mapToCurrentInstance(TruncateBoundary.YEAR, -1, month, day, hr, min, offsetMin);
- }
-
- private static String evaluateCurrent(String curExpr) throws Exception {
- if(curExpr.equals("")) {
- return curExpr;
- }
-
- int inst = CoordCommandUtils.parseOneArg(curExpr);
- return CoordELFunctions.ph2_coord_current(inst);
- }
-
- public static String ph2_now(int hr, int min) throws Exception {
- String inst = ph2_now_inst(hr, min);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_today(int hr, int min) throws Exception {
- String inst = ph2_today_inst(hr, min);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_todayWithOffset(int hr, int min, int offsetMin) throws Exception {
- String inst = ph2_todayWithOffset_inst(hr, min, offsetMin);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_yesterday(int hr, int min) throws Exception {
- String inst = ph2_yesterday_inst(hr, min);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_yesterdayWithOffset(int hr, int min, int offsetMin) throws Exception {
- String inst = ph2_yesterdayWithOffset_inst(hr, min, offsetMin);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_currentMonth(int day, int hr, int min) throws Exception {
- String inst = ph2_currentMonth_inst(day, hr, min);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_currentMonthWithOffset(int day, int hr, int min, int offsetMin) throws Exception {
- String inst = ph2_currentMonthWithOffset_inst(day, hr, min, offsetMin);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_lastMonth(int day, int hr, int min) throws Exception {
- String inst = ph2_lastMonth_inst(day, hr, min);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_lastMonthWithOffset(int day, int hr, int min, int offsetMin) throws Exception {
- String inst = ph2_lastMonthWithOffset_inst(day, hr, min, offsetMin);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_currentYear(int month, int day, int hr, int min) throws Exception {
- String inst = ph2_currentYear_inst(month, day, hr, min);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_currentYearWithOffset(int month, int day, int hr, int min, int offsetMIn) throws Exception {
- String inst = ph2_currentYearWithOffset_inst(month, day, hr, min, offsetMIn);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_lastYear(int month, int day, int hr, int min) throws Exception {
- String inst = ph2_lastYear_inst(month, day, hr, min);
- return evaluateCurrent(inst);
- }
-
- public static String ph2_lastYearWithOffset(int month, int day, int hr, int min, int offsetMin) throws Exception {
- String inst = ph2_lastYearWithOffset_inst(month, day, hr, min, offsetMin);
- return evaluateCurrent(inst);
- }
-
- /**
- * Maps the dataset time to coord:current(n) with respect to action's nominal time
- * dataset time = truncate(nominal time) + yr + day + month + hr + min
- * @param trunc : Truncate resolution
- * @param yr : Year to add (can be -ve)
- * @param month : month to add (can be -ve)
- * @param day : day to add (can be -ve)
- * @param hr : hr to add (can be -ve)
- * @param min : min to add (can be -ve)
- * @param offsetMinutes - minutes to offset from nominal time
- * @return coord:current(n)
- * @throws Exception : If encountered an exception while evaluating
- */
- //TODO handle the case where action_Creation_time or the n-th instance is earlier than the Initial_Instance of dataset.
- private static String mapToCurrentInstance(TruncateBoundary trunc, int yr, int month,
- int day, int hr, int min, int offsetMinutes) throws Exception {
- Calendar nominalInstanceCal = CoordELFunctions.getEffectiveNominalTime();
- if (nominalInstanceCal == null) {
- XLog.getLog(OozieELExtensions.class).warn(
- "If the initial instance of the dataset is later than the nominal time, an empty string is returned. " +
- "This means that no data is available at the current-instance specified by the user " +
- "and the user could try modifying his initial-instance to an earlier time.");
- return "";
- }
-
- Calendar dsInstanceCal = Calendar.getInstance(CoordELFunctions.getDatasetTZ());
- dsInstanceCal.setTime(nominalInstanceCal.getTime());
- dsInstanceCal.add(Calendar.MINUTE, offsetMinutes);
-
- //truncate
- switch (trunc) {
- case YEAR:
- dsInstanceCal.set(Calendar.MONTH, 0);
- case MONTH:
- dsInstanceCal.set(Calendar.DAY_OF_MONTH, 1);
- case DAY:
- dsInstanceCal.set(Calendar.HOUR_OF_DAY, 0);
- dsInstanceCal.set(Calendar.MINUTE, 0);
- dsInstanceCal.set(Calendar.SECOND, 0);
- dsInstanceCal.set(Calendar.MILLISECOND, 0);
- break;
- case NONE: //don't truncate
- break;
- default:
- throw new IllegalArgumentException("Truncation boundary " + trunc + " is not supported");
- }
-
- //add
- dsInstanceCal.add(Calendar.YEAR, yr);
- dsInstanceCal.add(Calendar.MONTH, month);
- dsInstanceCal.add(Calendar.DAY_OF_MONTH, day);
- dsInstanceCal.add(Calendar.HOUR_OF_DAY, hr);
- dsInstanceCal.add(Calendar.MINUTE, min);
-
- int[] instCnt = new int[1];
- Calendar compInstCal = CoordELFunctions.getCurrentInstance(dsInstanceCal.getTime(), instCnt);
- if(compInstCal == null) {
- return "";
- }
- int dsInstanceCnt = instCnt[0];
-
- compInstCal = CoordELFunctions.getCurrentInstance(nominalInstanceCal.getTime(), instCnt);
- if(compInstCal == null) {
- return "";
- }
- int nominalInstanceCnt = instCnt[0];
-
- return COORD_CURRENT + "(" + (dsInstanceCnt - nominalInstanceCnt) + ")";
- }
-}
View
44 src/test/java/org/apache/oozie/extensions/TestOozieELExtensions.java
@@ -20,6 +20,7 @@
import java.io.File;
+import org.apache.oozie.client.CoordinatorJob.Timeunit;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.coord.CoordELFunctions;
import org.apache.oozie.coord.SyncCoordAction;
@@ -82,23 +83,11 @@ public void testActionExpressions() throws Exception {
Assert.assertEquals(CoordELFunctions.evalAndWrap(eval, "${lastYear(0, 0, 0, 0)}"), "2008-01-01T00:00Z");
Assert.assertEquals(CoordELFunctions.evalAndWrap(eval, "${lastYear(1, 0, 0, 0)}"), "2008-02-01T00:00Z");
- }
-
- @Test
- public void testIntstanceTime() throws Exception {
- ELEvaluator eval = createActionStartEvaluator();
+
Assert.assertEquals(CoordELFunctions.evalAndWrap(eval, "${instanceTime()}"), "2009-09-02T10:00Z");
- }
-
- @Test
- public void testDateOffset() throws Exception {
- ELEvaluator eval = createActionStartEvaluator();
+
Assert.assertEquals(CoordELFunctions.evalAndWrap(eval, "${dateOffset(instanceTime(), 1, 'DAY')}"), "2009-09-03T10:00Z");
- }
-
- @Test
- public void testFormatTime() throws Exception {
- ELEvaluator eval = createActionStartEvaluator();
+
Assert.assertEquals(CoordELFunctions.evalAndWrap(eval, "${formatTime(instanceTime(), 'yyyy-MMM-dd')}"), "2009-Sep-02");
}
@@ -112,10 +101,25 @@ public void testUser() throws Exception {
@Test
public void testDataIn() throws Exception {
- ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
- String uris = "hdfs://localhost:8020/clicks/2010/01/01/00,hdfs://localhost:8020/clicks/2010/01/01/01";
- eval.setVariable(".datain.clicks", uris );
- String expuris = "hdfs://localhost:8020/clicks/2010/01/01/00/*/US,hdfs://localhost:8020/clicks/2010/01/01/01/*/US";
+ ELEvaluator eval = createActionStartEvaluator();
+ String uris = "hdfs://localhost:8020/clicks/2009/09/02/10,hdfs://localhost:8020/clicks/2009/09/02/09";
+ eval.setVariable(".datain.clicks", uris);
+ String expuris = "hdfs://localhost:8020/clicks/2009/09/02/10/*/US,hdfs://localhost:8020/clicks/2009/09/02/09/*/US";
+ Assert.assertEquals(expuris, CoordELFunctions.evalAndWrap(eval, "${dataIn('clicks', '*/US')}"));
+
+ //test optional input
+ String inName = "clicks";
+ SyncCoordDataset ds = createDataSet("2007-09-30T010:00Z");
+ eval.setVariable(inName + ".frequency", String.valueOf(ds.getFrequency()));
+ eval.setVariable(inName + ".freq_timeunit", ds.getTimeUnit().name());
+ eval.setVariable(inName + ".timezone", ds.getTimeZone().getID());
+ eval.setVariable(inName + ".end_of_duration", Timeunit.NONE.name());
+ eval.setVariable(inName + ".initial-instance", DateUtils.formatDateUTC(ds.getInitInstance()));
+ eval.setVariable(inName + ".done-flag", "notused");
+ eval.setVariable(inName + ".uri-template", ds.getUriTemplate());
+ eval.setVariable(inName + ".start-instance", "now(-1,0)");
+ eval.setVariable(inName + ".end-instance", "now(0,0)");
+ eval.setVariable(".datain.clicks", null);
Assert.assertEquals(expuris, CoordELFunctions.evalAndWrap(eval, "${dataIn('clicks', '*/US')}"));
}
@@ -236,7 +240,7 @@ private SyncCoordDataset createDataSet(String initialInstance) throws Exception
ds.setTimeUnit(TimeUnit.HOUR);
ds.setTimeZone(DateUtils.getTimeZone("UTC"));
ds.setName("test");
- ds.setUriTemplate("hdfs://localhost:9000/user/test_user/US/${YEAR}/${MONTH}/${DAY}");
+ ds.setUriTemplate("hdfs://localhost:8020/clicks/${YEAR}/${MONTH}/${DAY}/${HOUR}");
ds.setType("SYNC");
ds.setDoneFlag("");
return ds;
View
315 src/test/java/org/apache/oozie/extensions/TestOozieELExtensionsOld.java
@@ -1,315 +0,0 @@
-package org.apache.oozie.extensions;
-
-import java.io.File;
-
-import org.apache.oozie.client.OozieClient;
-import org.apache.oozie.coord.CoordELFunctions;
-import org.apache.oozie.coord.SyncCoordAction;
-import org.apache.oozie.coord.SyncCoordDataset;
-import org.apache.oozie.coord.TimeUnit;
-import org.apache.oozie.service.ConfigurationService;
-import org.apache.oozie.service.ELService;
-import org.apache.oozie.service.Services;
-import org.apache.oozie.util.DateUtils;
-import org.apache.oozie.util.ELEvaluator;
-import org.testng.Assert;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-public class TestOozieELExtensionsOld {
-
- private ELEvaluator instEval;
- private ELEvaluator createEval;
-
- @BeforeClass
- public void setUp() throws Exception {
- String curPath = new File(".").getAbsolutePath();
- System.setProperty(Services.OOZIE_HOME_DIR, curPath);
- String confPath = new File(getClass().getResource("/oozie-site-old.xml").getFile()).getParent();
- System.setProperty(ConfigurationService.OOZIE_CONFIG_DIR, confPath);
- System.setProperty(ConfigurationService.OOZIE_CONFIG_FILE, "oozie-site-old.xml");
- Services.setOozieHome();
-
- Services services = new Services();
- services.getConf().set("oozie.services", "org.apache.oozie.service.ELService");
- services.init();
-
- instEval = Services.get().get(ELService.class).createEvaluator("coord-action-create-inst");
- instEval.setVariable(OozieClient.USER_NAME, "test_user");
- instEval.setVariable(OozieClient.GROUP_NAME, "test_group");
- createEval = Services.get().get(ELService.class).createEvaluator("coord-action-create");
- createEval.setVariable(OozieClient.USER_NAME, "test_user");
- createEval.setVariable(OozieClient.GROUP_NAME, "test_group");
- }
-
- @Test
- public void testDataIn() throws Exception {
- ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
- String uris = "hdfs://localhost:8020/clicks/2010/01/01/00,hdfs://localhost:8020/clicks/2010/01/01/01";
- eval.setVariable(".datain.clicks", uris );
- String expuris = "hdfs://localhost:8020/clicks/2010/01/01/00/*/US,hdfs://localhost:8020/clicks/2010/01/01/01/*/US";
- Assert.assertEquals(expuris, CoordELFunctions.evalAndWrap(eval, "${elext:dataIn('clicks', '*/US')}"));
- }
-
- @Test
- public void testCurrentMonth() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:currentMonth(0,0,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-01T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:currentMonth(2,-1,0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T23:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:currentMonthWithOffset(2,-1,0,0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T23:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testCurrentMonthWithOffset() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:currentMonth(2,-1,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T23:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2009-10-02T16:30Z", "2009-10-02T16:30Z");
-
- expr = "${elext:currentMonthWithOffset(2,-1,0,-43200)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T23:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2009-09-03T16:30Z", "2009-09-03T16:30Z");
-
- expr = "${elext:currentMonthWithOffset(2,-1,0,-1440)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T23:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2009-09-30T23:00Z", "2009-09-30T22:00Z");
-
- expr = "${elext:currentMonth(0,0,0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-01T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2009-10-02T23:00Z", "2009-10-02T22:00Z");
-
- expr = "${elext:currentMonthWithOffset(0,0,0, -2880)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-01T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- private String getELExpression(String expr) {
- if(expr != null) {
- return "${" + expr + "}";
- }
- return null;
- }
-
- @Test
- public void testToday() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:today(0,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:today(1,-20)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:todayWithOffset(1,-20, 0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testTodayOffset() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:today(2,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T02:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2009-09-03T11:30Z", "2009-09-03T10:30Z");
-
- expr = "${elext:todayWithOffset(2,0,-1440)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T02:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2009-09-02T16:30Z", "2009-09-02T16:30Z");
-
- expr = "${elext:todayWithOffset(2,0,-240)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T02:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2009-09-03T16:30Z", "2009-09-03T15:30Z");
-
- expr = "${elext:todayWithOffset(2,0,-240)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertFalse("2009-09-02T02:00Z".equals(CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult))));
- }
-
- @Test
- public void testNow() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:now(0,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T10:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:now(2,-10)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-02T12:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testYesterday() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:yesterday(0,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-01T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:yesterday(1,10)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:yesterdayWithOffset(1,10, 0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testYesterdayWithOffset() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:yesterday(1,10)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2009-09-03T16:30Z", "2009-09-03T16:30Z");
-
- expr = "${elext:yesterdayWithOffset(1,10, -1440)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-09-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testLastMonth() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:lastMonth(0,0,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-08-01T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:lastMonth(1,1,10)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-08-02T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:lastMonthWithOffset(1,1,10,0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-08-02T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testLastMonthWithOffset() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:lastMonth(0,0,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-08-01T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2009-09-03T16:30Z", "2009-09-03T16:30Z");
-
- expr = "${elext:lastMonthWithOffset(0,0,0,-1440)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-08-01T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testCurrentYear() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:currentYear(0,0,0,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-01-01T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:currentYear(1,0,1,0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-02-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:currentYearWithOffset(1,0,1,0,0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-02-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testCurrentYearWithOffset() throws Exception {
- initForCurrentThread("2007-09-30T010:00Z", "2009-12-30T00:30Z", "2009-12-30T00:00Z");
-
- String expr = "${elext:currentYear(1,0,1,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-02-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- initForCurrentThread("2007-09-30T010:00Z", "2010-01-01T00:30Z", "2010-01-01T00:00Z");
-
- expr = "${elext:currentYearWithOffset(1,0,1,0,-2880)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2009-02-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testLastYear() throws Exception {
- initForCurrentThread();
-
- String expr = "${elext:lastYear(0,0,0,0)}";
- String instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2008-01-01T00:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:lastYear(1,0,1,0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2008-02-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
-
- expr = "${elext:lastYearWithOffset(1,0,1,0, 0)}";
- instResult = CoordELFunctions.evalAndWrap(instEval, expr);
- Assert.assertEquals("2008-02-01T01:00Z", CoordELFunctions.evalAndWrap(createEval, getELExpression(instResult)));
- }
-
- @Test
- public void testNominalGreaterThanInitial() throws Exception {
- initForCurrentThread("2009-08-30T010:00Z", "2009-09-02T11:30Z", "2009-09-02T10:30Z");
- String expr = "${elext:currentYear(0,0,0,0)}";
- Assert.assertEquals("", CoordELFunctions.evalAndWrap(instEval, expr));
- }
-
- private void initForCurrentThread() throws Exception {
- initForCurrentThread("2007-09-30T010:00Z", "2009-09-02T11:30Z", "2009-09-02T10:30Z");
- }
-
- private void initForCurrentThread(String initialInstance, String actualTime, String nominalTime) throws Exception {
- SyncCoordAction appInst;
- SyncCoordDataset ds;
- ds = new SyncCoordDataset();
- ds.setFrequency(1);
- ds.setInitInstance(DateUtils.parseDateUTC(initialInstance));
- ds.setTimeUnit(TimeUnit.HOUR);
- ds.setTimeZone(DateUtils.getTimeZone("UTC"));
- ds.setName("test");
- ds.setUriTemplate("hdfs://localhost:9000/user/test_user/US/${YEAR}/${MONTH}/${DAY}");
- ds.setType("SYNC");
- ds.setDoneFlag("");
-
- appInst = new SyncCoordAction();
- appInst.setActualTime(DateUtils.parseDateUTC(actualTime));
- appInst.setNominalTime(DateUtils.parseDateUTC(nominalTime));
- appInst.setTimeZone(DateUtils.getTimeZone("UTC"));
- appInst.setActionId("00000-oozie-C@1");
- appInst.setName("mycoordinator-app");
-
- CoordELFunctions.configureEvaluator(instEval, ds, appInst);
- CoordELFunctions.configureEvaluator(createEval, ds, appInst);
- }
-}
View
113 src/test/resources/oozie-site-old.xml
@@ -1,113 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
- <property>
- <name>oozie.service.ELService.ext.functions.coord-job-submit-instances</name>
- <value>
- elext:now=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_now_echo,
- elext:today=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_today_echo,
- elext:yesterday=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_yesterday_echo,
- elext:currentMonth=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_currentMonth_echo,
- elext:lastMonth=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_lastMonth_echo,
- elext:currentYear=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_currentYear_echo,
- elext:lastYear=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_lastYear_echo,
- elext:todayWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_todayWithOffset_echo,
- elext:yesterdayWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_yesterdayWithOffset_echo,
- elext:currentMonthWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_currentMonthWithOffset_echo,
- elext:lastMonthWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_lastMonthWithOffset_echo,
- elext:currentYearWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_currentYearWithOffset_echo,
- elext:lastYearWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_lastYearWithOffset_echo
- </value>
- <description>
- EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
- This property is a convenience property to add extensions to the built in executors without having to
- include all the built in ones.
- </description>
- </property>
- <property>
- <name>oozie.service.ELService.ext.functions.coord-action-create-inst</name>
- <value>
- elext:currentMonth=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_currentMonth_inst,
- elext:now=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_now_inst,
- elext:today=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_today_inst,
- elext:yesterday=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_yesterday_inst,
- elext:lastMonth=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_lastMonth_inst,
- elext:currentYear=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_currentYear_inst,
- elext:lastYear=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_lastYear_inst,
- elext:todayWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_todayWithOffset_inst,
- elext:yesterdayWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_yesterdayWithOffset_inst,
- elext:currentMonthWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_currentMonthWithOffset_inst,
- elext:lastMonthWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_lastMonthWithOffset_inst,
- elext:currentYearWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_currentYearWithOffset_inst,
- elext:lastYearWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_lastYearWithOffset_inst
- </value>
- <description>
- EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
- This property is a convenience property to add extensions to the built in executors without having to
- include all the built in ones.
- </description>
- </property>
- <property>
- <name>oozie.service.ELService.ext.functions.coord-action-create</name>
- <value>
- elext:currentMonth=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_currentMonth,
- elext:now=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_now,
- elext:today=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_today,
- elext:yesterday=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_yesterday,
- elext:currentMonth=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_currentMonth,
- elext:lastMonth=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_lastMonth,
- elext:currentYear=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_currentYear,
- elext:lastYear=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_lastYear,
- elext:todayWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_todayWithOffset,
- elext:yesterdayWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_yesterdayWithOffset,
- elext:currentMonthWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_currentMonthWithOffset,
- elext:lastMonthWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_lastMonthWithOffset,
- elext:currentYearWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_currentYearWithOffset,
- elext:lastYearWithOffset=org.apache.oozie.extensions.OozieELExtensionsOld#ph2_lastYearWithOffset
- </value>
- <description>
- EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
- This property is a convenience property to add extensions to the built in executors without having to
- include all the built in ones.
- </description>
- </property>
- <property>
- <name>oozie.service.ELService.ext.functions.coord-job-submit-data</name>
- <value>
- elext:dataIn=org.apache.oozie.extensions.OozieELExtensionsOld#ph1_dataIn_echo
- </value>
- <description>
- EL constant declarations, separated by commas, format is [PREFIX:]NAME=CLASS#CONSTANT.
- This property is a convenience property to add extensions to the built in executors without having to
- include all the built in ones.
- </description>
- </property>
- <property>
- <name>oozie.service.ELService.ext.functions.coord-action-start</name>
- <value>
- elext:dataIn=org.apache.oozie.extensions.OozieELExtensionsOld#ph3_dataIn
- </value>
- <description>
- EL functions declarations, separated by commas, format is [PREFIX:]NAME=CLASS#METHOD.
- This property is a convenience property to add extensions to the built in executors without having to
- include all the built in ones.
- </description>
- </property>
-</configuration>
Please sign in to comment.
Something went wrong with that request. Please try again.