From 3b7fd63149f874c0d533957ba9c313d9ff81cb4b Mon Sep 17 00:00:00 2001 From: Pragya Date: Thu, 11 Feb 2016 10:23:51 +0000 Subject: [PATCH 01/29] FALCON-1829 Add regression for submit and schedule process on native scheduler (time based) --- falcon-regression/CHANGES.txt | 2 + .../helpers/entity/AbstractEntityHelper.java | 52 ++-- .../falcon/regression/AuthorizationTest.java | 4 +- .../nativeScheduler/NativeScheduleTest.java | 231 ++++++++++++++++++ .../falcon/regression/security/EntityOp.java | 4 +- .../src/test/resources/sleep/workflow.xml | 85 +++++++ 6 files changed, 354 insertions(+), 24 deletions(-) create mode 100644 falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeScheduleTest.java create mode 100644 falcon-regression/merlin/src/test/resources/sleep/workflow.xml diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index b4717f4d5..b3769f031 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -5,6 +5,8 @@ Trunk (Unreleased) INCOMPATIBLE CHANGES NEW FEATURES + FALCON-1829 Add regression for submit and schedule process on native scheduler (time based) (Pragya Mittal) + FALCON-1766 Add CLI metrics check for HiveDR, HDFS and feed replication (Paul Isaychuk) FALCON-1777 Add regression for HDFS replication (recipe) (Paul Isaychuk) diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java index 29c97b2ff..27e12d00f 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java @@ -301,44 +301,55 @@ public ServiceResponse validateEntity(String data) public ServiceResponse submitEntity(String data, String user) throws IOException, URISyntaxException, AuthenticationException, InterruptedException { LOGGER.info("Submitting " + getEntityType() + ": \n" + Util.prettyPrintXml(data)); - return Util.sendRequest(createUrl(this.hostname + URLS.SUBMIT_URL.getValue(), - getEntityType() + colo), "post", data, user); + return Util.sendRequest(createUrl(this.hostname + URLS.SUBMIT_URL.getValue(), getEntityType() + colo), "post", + data, user); } public ServiceResponse validateEntity(String data, String user) throws IOException, URISyntaxException, AuthenticationException, InterruptedException { LOGGER.info("Validating " + getEntityType() + ": \n" + Util.prettyPrintXml(data)); - return Util.sendRequest(createUrl(this.hostname + URLS.VALIDATE_URL.getValue(), - getEntityType() + colo), "post", data, user); + return Util.sendRequest(createUrl(this.hostname + URLS.VALIDATE_URL.getValue(), getEntityType() + colo), "post", + data, user); } public ServiceResponse schedule(String processData) throws IOException, URISyntaxException, AuthenticationException, InterruptedException { - return schedule(processData, null); + return schedule(processData, null, ""); } - public ServiceResponse schedule(String processData, String user) + public ServiceResponse schedule(String data, String user, String params) throws IOException, URISyntaxException, AuthenticationException, InterruptedException { - return Util.sendRequest(createUrl(this.hostname + URLS.SCHEDULE_URL.getValue(), - getEntityType(), getEntityName(processData) + colo), "post", user); + + String url = createUrl(this.hostname + URLS.SCHEDULE_URL.getValue(), getEntityType(), + getEntityName(data) + colo); + if (StringUtils.isNotBlank(params)) { + url += (colo.isEmpty() ? "?" : "&") + params; + } + LOGGER.info("url is : " + url); + return Util.sendRequest(createUrl(url), "post", data, user); } public ServiceResponse submitAndSchedule(String data) throws IOException, URISyntaxException, AuthenticationException, InterruptedException { - return submitAndSchedule(data, null); + return submitAndSchedule(data, null, ""); } - public ServiceResponse submitAndSchedule(String data, String user) + public ServiceResponse submitAndSchedule(String data, String user, String params) throws IOException, URISyntaxException, AuthenticationException, InterruptedException { LOGGER.info("Submitting " + getEntityType() + ": \n" + Util.prettyPrintXml(data)); - return Util.sendRequest(createUrl(this.hostname + URLS.SUBMIT_AND_SCHEDULE_URL.getValue(), - getEntityType()), "post", data, user); + + String url = createUrl(this.hostname + URLS.SUBMIT_AND_SCHEDULE_URL.getValue(), getEntityType() + colo); + if (StringUtils.isNotBlank(params)) { + url += (colo.isEmpty() ? "?" : "&") + params; + } + return Util.sendRequest(createUrl(url), "post", data, user); } public ServiceResponse deleteByName(String entityName, String user) throws AuthenticationException, IOException, URISyntaxException, InterruptedException { - return Util.sendRequest(createUrl(this.hostname + URLS.DELETE_URL.getValue(), - getEntityType(), entityName + colo), "delete", user); + return Util.sendRequest( + createUrl(this.hostname + URLS.DELETE_URL.getValue(), getEntityType(), entityName + colo), "delete", + user); } public ServiceResponse delete(String data) @@ -348,8 +359,9 @@ public ServiceResponse delete(String data) public ServiceResponse delete(String data, String user) throws IOException, URISyntaxException, AuthenticationException, InterruptedException { - return Util.sendRequest(createUrl(this.hostname + URLS.DELETE_URL.getValue(), - getEntityType(), getEntityName(data) + colo), "delete", user); + return Util.sendRequest( + createUrl(this.hostname + URLS.DELETE_URL.getValue(), getEntityType(), getEntityName(data) + colo), + "delete", user); } public ServiceResponse suspend(String data) @@ -398,8 +410,9 @@ public ServiceResponse getEntityDefinition(String data, String user) public ServiceResponse getEntityDependencies(String data, String user) throws IOException, URISyntaxException, AuthenticationException, InterruptedException { - return Util.sendRequest(createUrl(this.hostname + URLS.DEPENDENCIES.getValue(), - getEntityType(), getEntityName(data) + colo), "get", user); + return Util.sendRequest( + createUrl(this.hostname + URLS.DEPENDENCIES.getValue(), getEntityType(), getEntityName(data) + colo), + "get", user); } public InstancesResult getRunningInstance(String name) @@ -661,8 +674,7 @@ public InstancesResult listInstances(String entityName, String params, String us */ public ServiceResponse getDependencies(String entityName) throws URISyntaxException, AuthenticationException, InterruptedException, IOException { - String url = createUrl(this.hostname + URLS.DEPENDENCIES.getValue(), getEntityType(), - entityName + colo); + String url = createUrl(this.hostname + URLS.DEPENDENCIES.getValue(), getEntityType(), entityName + colo); return Util.sendRequest(url, "get", null, null); } diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java index 9c37562b2..714a21f81 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/AuthorizationTest.java @@ -581,7 +581,7 @@ public void u1ScheduleFeedU2ScheduleDependantProcessU1UpdateFeed() throws Except //by U2 schedule process dependant on scheduled feed by U1 ServiceResponse serviceResponse = prism.getProcessHelper() - .submitAndSchedule(bundles[0].getProcessData(), MerlinConstants.USER2_NAME); + .submitAndSchedule(bundles[0].getProcessData(), MerlinConstants.USER2_NAME, ""); AssertUtil.assertSucceeded(serviceResponse); AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, bundles[0].getProcessData(), Job.Status.RUNNING); @@ -631,7 +631,7 @@ public void u1ScheduleFeedU2ScheduleDependantProcessU2UpdateFeed() throws Except //by U2 schedule process dependent on scheduled feed by U1 ServiceResponse serviceResponse = prism.getProcessHelper().submitAndSchedule(bundles[0].getProcessData(), - MerlinConstants.USER2_NAME); + MerlinConstants.USER2_NAME, ""); AssertUtil.assertSucceeded(serviceResponse); AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, bundles[0].getProcessData(), Job.Status.RUNNING); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeScheduleTest.java new file mode 100644 index 000000000..fe61cdf28 --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeScheduleTest.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.falcon.regression.nativeScheduler; +import org.apache.falcon.entity.v0.Frequency; +import org.apache.falcon.regression.Entities.ProcessMerlin; +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.response.ServiceResponse; +import org.apache.falcon.regression.core.util.*; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.log4j.Logger; +import org.apache.oozie.client.OozieClient; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Schedule process via native scheduler. + */ + +@Test(groups = "distributed") +public class NativeScheduleTest extends BaseTestClass { + private ColoHelper cluster1 = servers.get(0); + private ColoHelper cluster2 = servers.get(1); + private String baseTestHDFSDir = cleanAndGetTestDir(); + private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; + private static final Logger LOGGER = Logger.getLogger(NativeScheduleTest.class); + private String startTime; + private String endTime; + + + + @BeforeClass(alwaysRun = true) + public void uploadWorkflow() throws Exception { + uploadDirToClusters(aggregateWorkflowDir, OSUtil.concat(OSUtil.RESOURCES, "sleep")); + } + + @BeforeMethod(alwaysRun = true) + public void setUp() throws Exception { + startTime = TimeUtil.getTimeWrtSystemTime(-10); + endTime = TimeUtil.addMinsToTime(startTime, 50); + LOGGER.info("Time range between : " + startTime + " and " + endTime); + Bundle bundle = BundleUtil.readELBundle(); + + for (int i = 0; i < 2; i++) { + bundles[i] = new Bundle(bundle, servers.get(i)); + bundles[i].generateUniqueBundle(this); + bundles[i].setProcessWorkflow(aggregateWorkflowDir); + bundles[i].submitClusters(prism); + bundles[i].setProcessConcurrency(2); + bundles[i].setProcessValidity(startTime, endTime); + bundles[i].setProcessPeriodicity(1, Frequency.TimeUnit.minutes); + } + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + removeTestClassEntities(); + } + + + /** + * Successfully schedule process via native scheduler through prism and server on single cluster. + * Schedule the same process on oozie. It should fail. + */ + @Test + public void scheduleProcessWithNativeUsingProperties() throws Exception { + ProcessMerlin processMerlin = bundles[0].getProcessObject(); + processMerlin.setInputs(null); + processMerlin.setOutputs(null); + LOGGER.info(processMerlin.toString()); + + ServiceResponse response = prism.getProcessHelper().submitEntity(processMerlin.toString()); + AssertUtil.assertSucceeded(response); + + // Schedule with prism + response = prism.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertSucceeded(response); + + // Schedule with server + response = cluster1.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:oozie"); + AssertUtil.assertFailed(response); + + // Schedule with oozie via prism + response = prism.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:oozie"); + AssertUtil.assertFailed(response); + + // Schedule with oozie via server + response = cluster1.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertSucceeded(response); + + } + + /** + * Successfully schedule process via oozie scheduler (using properties) through prism and server on single cluster. + * Schedule the same process on native scheduler. It should fail. + */ + @Test + public void scheduleProcessWithOozieUsingProperties() throws Exception { + ProcessMerlin processMerlin = bundles[0].getProcessObject(); + processMerlin.setInputs(null); + processMerlin.setOutputs(null); + LOGGER.info(processMerlin.toString()); + + ServiceResponse response = prism.getProcessHelper().submitEntity(processMerlin.toString()); + AssertUtil.assertSucceeded(response); + + // Schedule with prism + response = prism.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:oozie"); + AssertUtil.assertSucceeded(response); + + // Schedule with server + response = cluster1.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:oozie"); + AssertUtil.assertSucceeded(response); + + // Schedule with native via server + response = cluster1.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertFailed(response); + + // Schedule with native via prism + response = prism.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertFailed(response); + + } + + /** + * Successfully schedule process via oozie scheduler(without properties) through prism and server on single cluster. + * Schedule the same process on native using properties. It should fail. + */ + @Test + public void scheduleProcessWithOozieWithNoParams() throws Exception { + ProcessMerlin processMerlin = bundles[0].getProcessObject(); + processMerlin.setInputs(null); + processMerlin.setOutputs(null); + LOGGER.info(processMerlin.toString()); + + ServiceResponse response = prism.getProcessHelper().submitEntity(processMerlin.toString()); + AssertUtil.assertSucceeded(response); + + // Schedule with prism + response = prism.getProcessHelper().schedule(processMerlin.toString(), null, ""); + AssertUtil.assertSucceeded(response); + + // Schedule with native via server + response = cluster1.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertFailed(response); + + // Schedule with native via prism + response = prism.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertFailed(response); + + } + + /** + * Successfully schedule process via native scheduler through prism and server on multiple cluster. + * Schedule the same process on oozie. It should fail. + */ + @Test(groups = {"prism", "0.2"}) + public void scheduleProcessWithNativeOnTwoClusters() throws Exception { + + ProcessMerlin processMerlinNative = bundles[0].getProcessObject(); + processMerlinNative.clearProcessCluster(); + processMerlinNative.addProcessCluster( + new ProcessMerlin.ProcessClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withValidity(startTime, endTime).build()); + processMerlinNative.addProcessCluster( + new ProcessMerlin.ProcessClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withValidity(startTime, endTime).build()); + processMerlinNative.setInputs(null); + processMerlinNative.setOutputs(null); + LOGGER.info(processMerlinNative.toString()); + + // Schedule with native via prism + ServiceResponse response = prism.getProcessHelper(). + submitAndSchedule(processMerlinNative.toString(), null, "properties=falcon.scheduler:native"); + AssertUtil.assertSucceeded(response); + + // Schedule with native via server1 + response = cluster1.getProcessHelper().schedule(processMerlinNative.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertSucceeded(response); + + // Schedule with native via server2 + response = cluster2.getProcessHelper().schedule(processMerlinNative.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertSucceeded(response); + + // Schedule with oozie via prism + response = prism.getProcessHelper().schedule(processMerlinNative.toString(), null, + "properties=falcon.scheduler:oozie"); + AssertUtil.assertFailed(response); + + // Schedule with oozie via server1 + response = cluster1.getProcessHelper().schedule(processMerlinNative.toString(), null, + "properties=falcon.scheduler:oozie"); + AssertUtil.assertFailed(response); + + // Schedule with oozie via server2 + response = cluster2.getProcessHelper().schedule(processMerlinNative.toString(), null, + "properties=falcon.scheduler:oozie"); + AssertUtil.assertFailed(response); + + } + +} diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/EntityOp.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/EntityOp.java index dbaae9b2c..7b03f326e 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/EntityOp.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/EntityOp.java @@ -163,7 +163,7 @@ public boolean executeAs(String user, AbstractEntityHelper helper, String data) public boolean executeAs(String user, AbstractEntityHelper helper, String data) { final ServiceResponse response; try { - response = helper.schedule(data, user); + response = helper.schedule(data, user, ""); } catch (IOException e) { logger.error("Caught exception: " + e); return false; @@ -207,7 +207,7 @@ public boolean executeAs(String user, AbstractEntityHelper helper, String data) public boolean executeAs(String user, AbstractEntityHelper helper, String data) { final ServiceResponse response; try { - response = helper.submitAndSchedule(data, user); + response = helper.submitAndSchedule(data, user, ""); } catch (IOException e) { logger.error("Caught exception: " + e); return false; diff --git a/falcon-regression/merlin/src/test/resources/sleep/workflow.xml b/falcon-regression/merlin/src/test/resources/sleep/workflow.xml new file mode 100644 index 000000000..bd7c82192 --- /dev/null +++ b/falcon-regression/merlin/src/test/resources/sleep/workflow.xml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + ${jobTracker} + ${nameNode} + + + + + + mapred.job.queue.name + ${queueName} + + + mapred.mapper.class + org.apache.hadoop.mapred.lib.IdentityMapper + + + mapred.reducer.class + org.apache.hadoop.mapred.lib.IdentityReducer + + + mapred.map.tasks + 1 + + + mapred.input.dir + ${nameNode}/tmp/falcon-regression/test + + + mapred.output.dir + ${nameNode}/tmp/falcon-regression/test/output/ + + + + + + + + + ${jobTracker} + ${nameNode} + org.apache.hadoop.mapreduce.SleepJob + -m + 1 + -mt + 60000 + -r + 0 + + + + + + + Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + From db094ead1b1e5f83c3bca5ab88ab0c4db9470e4e Mon Sep 17 00:00:00 2001 From: Pragya Date: Wed, 17 Feb 2016 09:59:20 +0000 Subject: [PATCH 02/29] FALCON-1832 Adding tags in SchedulableEntityInstance.compareTo() --- .../apache/falcon/resource/SchedulableEntityInstance.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java b/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java index 096873464..862584967 100644 --- a/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java +++ b/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java @@ -170,6 +170,11 @@ public int compareTo(SchedulableEntityInstance o) { return result; } + result = this.tags.compareTo(o.tags); + if (result != 0) { + return result; + } + return this.instanceTime.compareTo(o.instanceTime); } } From cd0664ae01fc8fb181c3fa9c51b396504c7c4b17 Mon Sep 17 00:00:00 2001 From: Pragya Date: Wed, 24 Feb 2016 10:51:41 +0000 Subject: [PATCH 03/29] Revert "FALCON-1832 Adding tags in SchedulableEntityInstance.compareTo()" This reverts commit db094ead1b1e5f83c3bca5ab88ab0c4db9470e4e. --- .../apache/falcon/resource/SchedulableEntityInstance.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java b/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java index 862584967..096873464 100644 --- a/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java +++ b/client/src/main/java/org/apache/falcon/resource/SchedulableEntityInstance.java @@ -170,11 +170,6 @@ public int compareTo(SchedulableEntityInstance o) { return result; } - result = this.tags.compareTo(o.tags); - if (result != 0) { - return result; - } - return this.instanceTime.compareTo(o.instanceTime); } } From b674558e610ee43d2093bf1d37e53f7d8f0192d3 Mon Sep 17 00:00:00 2001 From: Pragya Date: Wed, 24 Feb 2016 11:44:50 +0000 Subject: [PATCH 04/29] FALCON-1839 Test case for APIs for entities scheduled on native scheduler --- .../regression/core/util/InstanceUtil.java | 2 +- .../core/util/NativeInstanceUtil.java | 176 ++++++++++ .../NativeSchedulerAPITest.java | 306 ++++++++++++++++++ 3 files changed, 483 insertions(+), 1 deletion(-) create mode 100644 falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java create mode 100644 falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java index b07e27561..c31918327 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java @@ -657,7 +657,7 @@ public static void waitTillInstanceReachState(OozieClient client, String entityN * @param expectedStatus expected status we are waiting for * @return minutes to wait for expected status */ - private static int getMinutesToWait(EntityType entityType, CoordinatorAction.Status expectedStatus) { + public static int getMinutesToWait(EntityType entityType, CoordinatorAction.Status expectedStatus) { switch (expectedStatus) { case RUNNING: if (entityType == EntityType.PROCESS) { diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java new file mode 100644 index 000000000..606d74879 --- /dev/null +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.regression.core.util; + +import org.apache.falcon.entity.v0.Entity; +import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.Frequency; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.resource.InstancesResult; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.log4j.Logger; +import org.apache.oozie.client.CoordinatorAction; +import org.testng.Assert; + +import java.io.IOException; +import java.net.URISyntaxException; + +/** + * util functions related to instanceTest. + */ +public final class NativeInstanceUtil { + + public static final int INSTANCES_CREATED_TIMEOUT = OSUtil.IS_WINDOWS ? 20 : 10; + private static final Logger LOGGER = Logger.getLogger(NativeInstanceUtil.class); + + private NativeInstanceUtil() { + throw new AssertionError("Instantiating utility class..."); + } + + /** + * Waits till instance of specific entity will be created during timeout. + * Timeout is common for most of usual test cases. + * + * @param cluster ColoHelper - colo on which API to be executed + * @param entity definition of entity which describes job + * @param startTime start time of instance + * @param endTime end time of instance + */ + public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity, String startTime, String endTime) + throws InterruptedException, IOException, AuthenticationException, + URISyntaxException { + int sleep = INSTANCES_CREATED_TIMEOUT * 60 / 5; + waitTillInstancesAreCreated(cluster, entity, startTime, endTime, sleep); + } + + /** + * Waits till instances of specific job will be created during specific time. + * Use this method directly in unusual test cases where timeouts are different from trivial. + * In other cases use waitTillInstancesAreCreated(ColoHelper,Entity,String,String) + * + * @param cluster ColoHelper - colo on which API to be executed + * @param entity definition of entity which describes job + * @param startTime start time of instance + * @param endTime end time of instance + * @param totalMinutesToWait total time(in minutes) to wait for instance creation + */ + public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity, String startTime, + String endTime, int totalMinutesToWait) + throws URISyntaxException, AuthenticationException, InterruptedException, IOException { + String entityName = entity.getName(); + EntityType type = entity.getEntityType(); + String params = "?start=" + startTime; + params += (endTime.isEmpty() ? "" : "&end=" + endTime); + + for (int sleepCount = 0; sleepCount < totalMinutesToWait; sleepCount++) { + InstancesResult statusResult = cluster.getProcessHelper().getProcessInstanceStatus(entityName, params); + if (statusResult.getInstances() != null) { + break; + } + LOGGER.info(type + " " + entityName + " still doesn't have instance created"); + TimeUtil.sleepSeconds(5); + } + } + + /** + * Waits till given instance of process/feed reach expected state during specific time. + * + * @param cluster ColoHelper - colo on which API to be executed + * @param entity definition of entity which describes job + * @param instanceTime time of instance + * @param expectedStatus expected status we are waiting for + * @param frequency frequency of process/feed + */ + public static void waitTillInstanceReachState(ColoHelper cluster, Entity entity, String instanceTime, + CoordinatorAction.Status expectedStatus, Frequency frequency) + throws InterruptedException, IOException, AuthenticationException, URISyntaxException { + int totalMinutesToWait = InstanceUtil.getMinutesToWait(entity.getEntityType(), expectedStatus); + waitTillInstanceReachState(cluster, entity, instanceTime, expectedStatus, frequency, totalMinutesToWait); + } + + /** + * Waits till given instance of process/feed reach expected state during + * specific time. + * + * @param cluster ColoHelper - colo on which API to be executed + * @param entity definition of entity which describes job + * @param instanceTime time of instance + * @param expectedStatus expected status we are waiting for + * @param frequency frequency of process/feed + * @param totalMinutesToWait total time(in minutes) to wait + */ + public static void waitTillInstanceReachState(ColoHelper cluster, Entity entity, String instanceTime, + CoordinatorAction.Status expectedStatus, Frequency frequency, int totalMinutesToWait) + throws URISyntaxException, AuthenticationException, InterruptedException, IOException { + String entityName = entity.getName(); + EntityType type = entity.getEntityType(); + + String endTime=getNextInstanceTime(instanceTime, frequency); + String params = "?start=" + instanceTime + "&end=" + endTime; + + int maxTries = 50; + int totalSleepTime = totalMinutesToWait * 60; + int sleepTime = totalSleepTime / maxTries; + LOGGER.info(String.format("Sleep for %d seconds", sleepTime)); + for (int i = 0; i < maxTries; i++) { + InstancesResult statusResult = cluster.getProcessHelper().getProcessInstanceStatus(entityName, params); + if (statusResult.getInstances() != null) { + if (statusResult.getInstances()[0].getStatus().name().equals(expectedStatus.toString())) { + return; + } + } + LOGGER.info(type + " " + entityName + " still doesn't have expected status"); + TimeUtil.sleepSeconds(5); + } + Assert.fail("expected state of instance was never reached"); + } + + /** + * Returns the time of next instance for a given instanceTime. + * + * @param instanceTime time of instance + * @param frequency frequency of process/feed + */ + public static String getNextInstanceTime(String instanceTime, Frequency frequency) { + String nextInstanceTime; + int minsToAdd = 1; + Frequency.TimeUnit timeUnit = frequency.getTimeUnit(); + + switch (timeUnit) { + case minutes: + minsToAdd = frequency.getFrequencyAsInt(); + break; + case hours: + minsToAdd = frequency.getFrequencyAsInt()*60; + break; + case days: + minsToAdd = frequency.getFrequencyAsInt()*60*24; + break; + case months: + minsToAdd = frequency.getFrequencyAsInt()*60*24*30; + break; + default: + Assert.fail("Unexpected freqType = " + frequency); + break; + } + nextInstanceTime = TimeUtil.addMinsToTime(instanceTime, minsToAdd); + return nextInstanceTime; + } + +} diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java new file mode 100644 index 000000000..1ef72b379 --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java @@ -0,0 +1,306 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.falcon.regression.nativeScheduler; +import org.apache.falcon.entity.v0.Frequency; +import org.apache.falcon.regression.Entities.ProcessMerlin; +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.response.ServiceResponse; +import org.apache.falcon.regression.core.util.AssertUtil; +import org.apache.falcon.regression.core.util.BundleUtil; +import org.apache.falcon.regression.core.util.OSUtil; +import org.apache.falcon.regression.core.util.TimeUtil; +import org.apache.falcon.regression.core.util.XmlUtil; +import org.apache.falcon.regression.core.util.Util; +import org.apache.falcon.regression.core.util.InstanceUtil; +import org.apache.falcon.regression.core.util.NativeInstanceUtil; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.falcon.resource.EntityList; +import org.apache.falcon.resource.EntitySummaryResult; +import org.apache.falcon.resource.InstanceDependencyResult; +import org.apache.falcon.resource.InstancesResult; +import org.apache.falcon.resource.TriageResult; +import org.apache.oozie.client.CoordinatorAction; +import org.apache.log4j.Logger; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Schedule process via native scheduler and test available APIs. + */ + +@Test(groups = {"distributed", "embedded" }) +public class NativeSchedulerAPITest extends BaseTestClass { + private ColoHelper cluster1 = servers.get(0); + private String baseTestHDFSDir = cleanAndGetTestDir(); + private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; + private static final Logger LOGGER = Logger.getLogger(NativeScheduleTest.class); + private ProcessMerlin processMerlin; + private String startTime; + private String endTime; + + @BeforeClass(alwaysRun = true) + public void uploadWorkflow() throws Exception { + uploadDirToClusters(aggregateWorkflowDir, OSUtil.concat(OSUtil.RESOURCES, "sleep")); + } + + @BeforeMethod(alwaysRun = true) + public void setUp() throws Exception { + startTime = TimeUtil.getTimeWrtSystemTime(-10); + endTime = TimeUtil.addMinsToTime(startTime, 50); + LOGGER.info("Time range between : " + startTime + " and " + endTime); + Bundle bundle = BundleUtil.readELBundle(); + + bundles[0] = new Bundle(bundle, servers.get(0)); + bundles[0].generateUniqueBundle(this); + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + bundles[0].submitClusters(prism); + bundles[0].setProcessConcurrency(2); + bundles[0].setProcessValidity(startTime, endTime); + bundles[0].setProcessPeriodicity(1, Frequency.TimeUnit.minutes); + + //Setting input and output feed as null (time based scheduling) + processMerlin = bundles[0].getProcessObject(); + processMerlin.setInputs(null); + processMerlin.setOutputs(null); + LOGGER.info(processMerlin.toString()); + + // Submit process + ServiceResponse response = prism.getProcessHelper().submitEntity(processMerlin.toString()); + AssertUtil.assertSucceeded(response); + + // Schedule with prism + response = prism.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertSucceeded(response); + + // Schedule with server + response = cluster1.getProcessHelper().schedule(processMerlin.toString(), null, + "properties=falcon.scheduler:native"); + AssertUtil.assertSucceeded(response); + + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + removeTestClassEntities(); + } + + /*Suspend and resume entity and check that it is successful. + */ + @Test(enabled = true) + public void entitySuspendResumeTest() throws Exception { + //Suspend entity via prism + AssertUtil.assertSucceeded(prism.getProcessHelper().suspend(processMerlin.toString())); + + Assert.assertTrue(Util.parseResponse(prism.getProcessHelper() + .getStatus(processMerlin.toString())).getMessage() + .contains("SUSPENDED")); + + //Resume entity via prism + AssertUtil.assertSucceeded(prism.getProcessHelper().resume(processMerlin.toString())); + + Assert.assertTrue(Util.parseResponse(prism.getProcessHelper() + .getStatus(processMerlin.toString())).getMessage() + .contains("RUNNING")); + + //Suspend entity via server + AssertUtil.assertSucceeded(cluster1.getProcessHelper().suspend(processMerlin.toString())); + + Assert.assertTrue(Util.parseResponse(cluster1.getProcessHelper() + .getStatus(processMerlin.toString())).getMessage() + .contains("SUSPENDED")); + + //Resume entity via server + AssertUtil.assertSucceeded(cluster1.getProcessHelper().resume(processMerlin.toString())); + + Assert.assertTrue(Util.parseResponse(cluster1.getProcessHelper() + .getStatus(processMerlin.toString())).getMessage() + .contains("RUNNING")); + } + + /*Test for entity definition, list, dependency, update and summary API and check that it is successful. + */ + @Test(enabled = true) + public void entityDefinitionListDependencyUpdateSummaryTest() throws Exception { + // Entity Definition + String processDef = prism.getProcessHelper().getEntityDefinition(processMerlin.toString()).getMessage(); + Assert.assertTrue(XmlUtil.isIdentical(processMerlin.toString(), processDef), "Definitions are not equal."); + + // Entity List + EntityList.EntityElement[] entityList = prism.getProcessHelper().listAllEntities().getEntityList().getElements(); + Assert.assertTrue(entityList.length==1); + Assert.assertTrue(entityList[0].type.equals("PROCESS")); + Assert.assertTrue(entityList[0].name.equals(processMerlin.getName())); + + // Entity Dependency + EntityList.EntityElement[] entityDependency = prism.getProcessHelper().getDependencies( + processMerlin.getName()).getEntityList().getElements(); + Assert.assertTrue(entityDependency.length==1); + Assert.assertTrue(entityDependency[0].type.equals("cluster")); + Assert.assertTrue(entityDependency[0].name.equals(bundles[0].getClusterElement().getName())); + + // Entity Update + NativeInstanceUtil.waitTillInstanceReachState(prism, processMerlin, startTime, CoordinatorAction.Status.RUNNING, + processMerlin.getFrequency()); + processMerlin.setParallel(3); + LOGGER.info("Updated process xml: " + processMerlin.toString()); + AssertUtil.assertSucceeded(prism.getProcessHelper().update(processMerlin.toString(), processMerlin.toString())); + processDef = prism.getProcessHelper().getEntityDefinition(processMerlin.toString()).getMessage(); + Assert.assertTrue(XmlUtil.isIdentical(processMerlin.toString(), processDef), "Definitions are not equal."); + + // Entity summary + EntitySummaryResult.EntitySummary[] summaries = cluster1.getProcessHelper() + .getEntitySummary(processMerlin.getClusterNames().get(0), "filterBy=STATUS:RUNNING") + .getEntitySummaryResult().getEntitySummaries(); + Assert.assertEquals(summaries.length, 1, "There should be one RUNNING process filtered."); + Assert.assertEquals(summaries[0].getName(), processMerlin.getName(), + "Summary shows invalid suspended process."); + } + + /*Test for instance dependency, list, params, logs, running and triage API and check that it is successful. + */ + @Test(enabled = true) + public void instanceAPITest() throws Exception { + // Instance dependency + NativeInstanceUtil.waitTillInstanceReachState(prism, processMerlin, startTime, CoordinatorAction.Status.RUNNING, + processMerlin.getFrequency()); + InstanceDependencyResult dependencyResult = prism.getProcessHelper().getInstanceDependencies( + processMerlin.getName(), "?instanceTime=" + startTime, null); + AssertUtil.assertSucceeded(dependencyResult); + Assert.assertNull(dependencyResult.getDependencies()); + + // Instance List + NativeInstanceUtil.waitTillInstanceReachState(prism, processMerlin, TimeUtil.addMinsToTime(startTime, 2), + CoordinatorAction.Status.SUCCEEDED, processMerlin.getFrequency()); + InstancesResult instanceResult = prism.getProcessHelper().listInstances(processMerlin.getName(), + "start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 2), null); + InstanceUtil.validateResponse(instanceResult, 2, 0, 0, 0, 0); + + // Instance Params + InstancesResult paramsResult = prism.getProcessHelper() + .getInstanceParams(processMerlin.getName(), "?start=" + startTime); + AssertUtil.assertSucceeded(paramsResult); + Assert.assertEquals(paramsResult.getInstances().length, 1); + Assert.assertEquals(paramsResult.getInstances()[0].getInstance(), startTime); + + // Instance logs + InstancesResult logsResult = prism.getProcessHelper() + .getProcessInstanceLogs(processMerlin.getName(), + "?start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 1)); + AssertUtil.assertSucceeded(logsResult); + Assert.assertEquals(logsResult.getInstances().length, 1); + Assert.assertEquals(logsResult.getInstances()[0].getInstance(), startTime); + + // Instance running + InstancesResult runningInstance = prism.getProcessHelper().getRunningInstance(processMerlin.getName()); + InstanceUtil.validateResponse(runningInstance, 2, 2, 0, 0, 0); + + // Instance triage + TriageResult responseTriage = prism.getProcessHelper().getInstanceTriage(processMerlin.getName(), + "?start=" + startTime); + AssertUtil.assertSucceeded(responseTriage); + Assert.assertEquals(responseTriage.getTriageGraphs().length, 1); + //There'll be just one process instance vertex and no edges + Assert.assertEquals(responseTriage.getTriageGraphs()[0].getVertices().length, 1); + } + + /*Test for instance suspend, status and resume API and check that it is successful. + */ + @Test + public void instanceSuspendResumeAPITest() throws Exception { + + String range = "?start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 5); + NativeInstanceUtil.waitTillInstanceReachState(prism, processMerlin, startTime, CoordinatorAction.Status.RUNNING, + processMerlin.getFrequency()); + // Suspend instance + InstancesResult instancesResult = prism.getProcessHelper().getProcessInstanceSuspend( + processMerlin.getName(), range); + InstanceUtil.validateResponse(instancesResult, 5, 0, 5, 0, 0); + + // Suspend again + instancesResult = prism.getProcessHelper().getProcessInstanceSuspend( + processMerlin.getName(), range); + Assert.assertNull(instancesResult.getInstances()); + + // Validate instance status + instancesResult = prism.getProcessHelper().getProcessInstanceStatus(processMerlin.getName(), + range); + InstanceUtil.validateResponse(instancesResult, 5, 0, 5, 0, 0); + + // Resume instance + instancesResult = prism.getProcessHelper().getProcessInstanceResume(processMerlin.getName(), + range); + InstanceUtil.validateResponse(instancesResult, 5, 2, 0, 0, 0); + + // Resume again + instancesResult = prism.getProcessHelper().getProcessInstanceResume(processMerlin.getName(), + range); + Assert.assertNull(instancesResult.getInstances()); + + // Validate instance status + instancesResult = prism.getProcessHelper().getProcessInstanceStatus(processMerlin.getName(), + range); + InstanceUtil.validateResponse(instancesResult, 5, 2, 0, 0, 0); + } + + /*Test for instance kill and rerun and check that it is successful. + */ + @Test + public void instanceKillRerunAPITest() throws Exception { + String range = "?start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 5); + NativeInstanceUtil.waitTillInstanceReachState(prism, processMerlin, startTime, CoordinatorAction.Status.RUNNING, + processMerlin.getFrequency()); + + //Instance Kill + InstancesResult instancesResult = prism.getProcessHelper().getProcessInstanceKill(processMerlin.getName(), range); + InstanceUtil.validateResponse(instancesResult, 5, 0, 0, 0, 5); + + // Instance rerun + prism.getProcessHelper().getProcessInstanceRerun(processMerlin.getName(), + "?start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 3)); + NativeInstanceUtil.waitTillInstanceReachState(prism, processMerlin, startTime, CoordinatorAction.Status.RUNNING, + processMerlin.getFrequency()); + + instancesResult = prism.getProcessHelper().getProcessInstanceStatus(processMerlin.getName(), range); + InstanceUtil.validateResponse(instancesResult, 5, 2, 0, 0, 2); + + } + + /*Test for rerun of succeeded instance and check that it is successful. + */ + @Test + public void instanceSucceedRerunAPITest() throws Exception { + String range = "?start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 3); + NativeInstanceUtil.waitTillInstanceReachState(prism, processMerlin, TimeUtil.addMinsToTime(startTime, 3), + CoordinatorAction.Status.SUCCEEDED, processMerlin.getFrequency()); + + // Rerun succeeded instance + prism.getProcessHelper().getProcessInstanceRerun(processMerlin.getName(), range + "&force=true"); + NativeInstanceUtil.waitTillInstanceReachState(prism, processMerlin, startTime, CoordinatorAction.Status.RUNNING, + processMerlin.getFrequency()); + + InstancesResult instanceResult = prism.getProcessHelper().getProcessInstanceStatus( + processMerlin.getName(), range); + InstanceUtil.validateResponse(instanceResult, 3, 2, 0, 0, 0); + } + +} From bbf551ac62c2bd0ebc66873621fb32bd1b30a1d7 Mon Sep 17 00:00:00 2001 From: Pragya Date: Thu, 25 Feb 2016 09:25:19 +0000 Subject: [PATCH 05/29] Review commenst addressed --- .../falcon/regression/core/util/NativeInstanceUtil.java | 6 +++--- .../regression/nativeScheduler/NativeSchedulerAPITest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java index 606d74879..97f01ab54 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java @@ -55,7 +55,7 @@ private NativeInstanceUtil() { public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity, String startTime, String endTime) throws InterruptedException, IOException, AuthenticationException, URISyntaxException { - int sleep = INSTANCES_CREATED_TIMEOUT * 60 / 5; + int sleep = INSTANCES_CREATED_TIMEOUT; waitTillInstancesAreCreated(cluster, entity, startTime, endTime, sleep); } @@ -84,7 +84,7 @@ public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity break; } LOGGER.info(type + " " + entityName + " still doesn't have instance created"); - TimeUtil.sleepSeconds(5); + TimeUtil.sleepSeconds(60); } } @@ -136,7 +136,7 @@ public static void waitTillInstanceReachState(ColoHelper cluster, Entity entity, } } LOGGER.info(type + " " + entityName + " still doesn't have expected status"); - TimeUtil.sleepSeconds(5); + TimeUtil.sleepSeconds(sleepTime); } Assert.fail("expected state of instance was never reached"); } diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java index 1ef72b379..f8a6eb328 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java @@ -52,7 +52,7 @@ public class NativeSchedulerAPITest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); private String baseTestHDFSDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; - private static final Logger LOGGER = Logger.getLogger(NativeScheduleTest.class); + private static final Logger LOGGER = Logger.getLogger(NativeSchedulerAPITest.class); private ProcessMerlin processMerlin; private String startTime; private String endTime; From a1a15afca62acbb317892bd6ad95ca9009fc2687 Mon Sep 17 00:00:00 2001 From: Pragya Date: Mon, 29 Feb 2016 07:37:07 +0000 Subject: [PATCH 06/29] FALCON-1841 Grouping test in falcon for running nightly regression --- .../apache/falcon/regression/ELExpCurrentAndLastWeekTest.java | 2 +- .../apache/falcon/regression/ELExpFutureAndLatestTest.java | 2 +- .../java/org/apache/falcon/regression/ELValidationsTest.java | 2 +- .../org/apache/falcon/regression/EmbeddedPigScriptTest.java | 2 +- .../org/apache/falcon/regression/FeedInstanceListingTest.java | 2 +- .../org/apache/falcon/regression/FeedInstanceStatusTest.java | 2 +- .../java/org/apache/falcon/regression/FeedLateRerunTest.java | 2 +- .../org/apache/falcon/regression/FeedReplicationTest.java | 2 +- .../java/org/apache/falcon/regression/FeedResumeTest.java | 2 +- .../test/java/org/apache/falcon/regression/FeedSLATest.java | 2 +- .../java/org/apache/falcon/regression/FeedScheduleTest.java | 2 +- .../java/org/apache/falcon/regression/FeedStatusTest.java | 2 +- .../apache/falcon/regression/FeedSubmitAndScheduleTest.java | 2 +- .../java/org/apache/falcon/regression/FeedSubmitTest.java | 2 +- .../java/org/apache/falcon/regression/FeedSuspendTest.java | 2 +- .../java/org/apache/falcon/regression/InstanceParamTest.java | 2 +- .../org/apache/falcon/regression/InstanceSummaryTest.java | 4 ++-- .../test/java/org/apache/falcon/regression/LogMoverTest.java | 2 +- .../org/apache/falcon/regression/ProcessFrequencyTest.java | 2 +- .../falcon/regression/ProcessInstanceColoMixedTest.java | 2 +- .../apache/falcon/regression/ProcessInstanceKillsTest.java | 2 +- .../apache/falcon/regression/ProcessInstanceRerunTest.java | 2 +- .../apache/falcon/regression/ProcessInstanceResumeTest.java | 2 +- .../apache/falcon/regression/ProcessInstanceRunningTest.java | 2 +- .../apache/falcon/regression/ProcessInstanceStatusTest.java | 2 +- .../apache/falcon/regression/ProcessInstanceSuspendTest.java | 2 +- .../org/apache/falcon/regression/ProcessLateRerunTest.java | 2 +- .../org/apache/falcon/regression/ProcessLibPathLoadTest.java | 2 +- .../java/org/apache/falcon/regression/ProcessLibPathTest.java | 2 +- .../java/org/apache/falcon/regression/ProcessSLATest.java | 2 +- .../java/org/apache/falcon/regression/ProcessUpdateTest.java | 2 +- .../apache/falcon/regression/TouchAPIPrismAndServerTest.java | 2 +- .../falcon/regression/ValidateAPIPrismAndServerTest.java | 2 +- .../falcon/regression/entity/EntitiesPatternSearchTest.java | 2 +- .../org/apache/falcon/regression/entity/ListEntitiesTest.java | 2 +- .../apache/falcon/regression/lineage/EntityLineageTest.java | 2 +- .../falcon/regression/lineage/ListFeedInstancesTest.java | 2 +- .../falcon/regression/lineage/ListProcessInstancesTest.java | 2 +- .../apache/falcon/regression/lineage/ProcessPipelineTest.java | 2 +- .../org/apache/falcon/regression/prism/EntityDryRunTest.java | 2 +- .../org/apache/falcon/regression/prism/OptionalInputTest.java | 2 +- .../falcon/regression/prism/PrismConcurrentRequestTest.java | 2 +- .../regression/prism/PrismFeedReplicationUpdateTest.java | 2 +- .../apache/falcon/regression/prism/PrismFeedScheduleTest.java | 2 +- .../org/apache/falcon/regression/prism/PrismFeedSnSTest.java | 1 + .../apache/falcon/regression/prism/PrismFeedSuspendTest.java | 1 + .../apache/falcon/regression/prism/PrismFeedUpdateTest.java | 2 +- .../falcon/regression/prism/PrismProcessScheduleTest.java | 1 + .../falcon/regression/prism/PrismProcessSuspendTest.java | 1 + .../regression/prism/ProcessPartitionExpVariableTest.java | 2 +- .../falcon/regression/prism/RescheduleKilledProcessTest.java | 2 +- .../regression/prism/RescheduleProcessInFinalStatesTest.java | 2 +- .../org/apache/falcon/regression/prism/RetentionTest.java | 2 +- .../falcon/regression/prism/UpdateAtSpecificTimeTest.java | 1 + .../falcon/regression/triage/FeedInstanceDependencyTest.java | 2 +- .../regression/triage/PipelineInstanceDependencyTest.java | 2 +- .../regression/triage/ProcessInstanceDependencyTest.java | 2 +- .../falcon/regression/triage/TriageAPISingleColoTest.java | 2 +- 58 files changed, 59 insertions(+), 54 deletions(-) diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpCurrentAndLastWeekTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpCurrentAndLastWeekTest.java index 3eb7bed9e..9d04ca38e 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpCurrentAndLastWeekTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpCurrentAndLastWeekTest.java @@ -49,7 +49,7 @@ * EL Expression Current and last week test. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class ELExpCurrentAndLastWeekTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpFutureAndLatestTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpFutureAndLatestTest.java index 90826f12c..49dfcc04b 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpFutureAndLatestTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELExpFutureAndLatestTest.java @@ -43,7 +43,7 @@ /** * EL Expression test. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class ELExpFutureAndLatestTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java index a0922cb1d..86936c82a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ELValidationsTest.java @@ -53,7 +53,7 @@ /** * EL Validations tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ELValidationsTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java index c49c381d6..564100ef7 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/EmbeddedPigScriptTest.java @@ -52,7 +52,7 @@ /** * Embedded pig script test. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class EmbeddedPigScriptTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceListingTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceListingTest.java index ecb57985a..88cb472b9 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceListingTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceListingTest.java @@ -50,7 +50,7 @@ /** * Test for https://issues.apache.org/jira/browse/FALCON-761. */ -@Test(groups = "embedded", timeOut = 900000) +@Test(groups = { "distributed", "embedded" }, timeOut = 900000) public class FeedInstanceListingTest extends BaseTestClass{ private String baseTestDir = cleanAndGetTestDir(); private String aggregateWorkflowDir = baseTestDir + "/aggregator"; diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java index 514fd103c..8b9ae70cc 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedInstanceStatusTest.java @@ -43,7 +43,7 @@ /** * Feed instance status tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedInstanceStatusTest extends BaseTestClass { private String baseTestDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java index 5bb5e6ed7..ec2b87798 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedLateRerunTest.java @@ -56,7 +56,7 @@ * On adding further late data it checks whether the data has been replicated correctly in the given late cut-off time. * Assuming that late frequency set in server is 3 minutes. Although value can be changed according to requirement. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedLateRerunTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java index a936aa111..dad0dc2d4 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java @@ -60,7 +60,7 @@ * feed replication test. * Replicates empty directories as well as directories containing data. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedReplicationTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java index ec117d736..2e6456948 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedResumeTest.java @@ -38,7 +38,7 @@ /** * Feed resume tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedResumeTest extends BaseTestClass { private final AbstractEntityHelper feedHelper = prism.getFeedHelper(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSLATest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSLATest.java index 28ddbd72d..088e98f2a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSLATest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSLATest.java @@ -39,7 +39,7 @@ /** * Feed SLA tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedSLATest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java index 79b722a54..5a9d47350 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedScheduleTest.java @@ -35,7 +35,7 @@ /** * Feed schedule tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedScheduleTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java index d5e8696f0..9abe1f214 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedStatusTest.java @@ -39,7 +39,7 @@ /** * Feed status tests. Checks getStatus functionality. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedStatusTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java index f7bf0f85f..e5ed322b2 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitAndScheduleTest.java @@ -43,7 +43,7 @@ /** * Feed submit and schedule tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedSubmitAndScheduleTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java index 53f3df85e..cf992f525 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSubmitTest.java @@ -36,7 +36,7 @@ /** * Feed submission tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedSubmitTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java index 521781823..953be0924 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedSuspendTest.java @@ -35,7 +35,7 @@ /** * Feed suspend tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class FeedSuspendTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java index 33808bfe3..779243b97 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceParamTest.java @@ -49,7 +49,7 @@ /** * tests for instance option params. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class InstanceParamTest extends BaseTestClass { /** diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java index 137491dd9..726a6ed64 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java @@ -54,7 +54,7 @@ 3. feed : same as test 1 for feed 4. feed : same as test 2 for feed */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class InstanceSummaryTest extends BaseTestClass { private String baseTestHDFSDir = cleanAndGetTestDir(); @@ -174,7 +174,7 @@ public void testSummarySingleClusterProcess() * Adjust multi-cluster process. Submit and schedule it. Get its instances summary. */ @Test(enabled = true, timeOut = 1200000) - public void testSummaryMultiClusterProcess() throws JAXBException, + public void testSummaryMultiClusterProcess() throws JAXBException, ParseException, IOException, URISyntaxException, AuthenticationException, InterruptedException { processBundle.setProcessValidity(startTime, endTime); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/LogMoverTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/LogMoverTest.java index f93630575..cb194f8a4 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/LogMoverTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/LogMoverTest.java @@ -49,7 +49,7 @@ * It is not working for map-reduce actions(FALCON-1038). * Using pig-action to test this feature. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class LogMoverTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessFrequencyTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessFrequencyTest.java index b0480e91c..77bd62fbf 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessFrequencyTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessFrequencyTest.java @@ -46,7 +46,7 @@ /** * Test process with different frequency combinations. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessFrequencyTest extends BaseTestClass { private static final Logger LOGGER = Logger.getLogger(ProcessFrequencyTest.class); private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java index 91d39a7e9..20f8f465e 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceColoMixedTest.java @@ -48,7 +48,7 @@ /** * Process instance mixed colo tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class ProcessInstanceColoMixedTest extends BaseTestClass { private final String baseTestHDFSDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java index 48c8021fc..71b39695f 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceKillsTest.java @@ -49,7 +49,7 @@ /** * Process instance kill tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessInstanceKillsTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java index d5b4ef2f3..ca0e84945 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRerunTest.java @@ -50,7 +50,7 @@ /** * Test Suite for instance rerun. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessInstanceRerunTest extends BaseTestClass { private boolean restartRequired; diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java index b7f4428dd..ebc06d6ec 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceResumeTest.java @@ -43,7 +43,7 @@ /** * Process instance resume tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessInstanceResumeTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java index 98fdcca50..97250b3f0 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceRunningTest.java @@ -47,7 +47,7 @@ /** * Regression for instance running api. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessInstanceRunningTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java index 14ecfe48b..e20df64c3 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java @@ -56,7 +56,7 @@ /** * Process instance status tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessInstanceStatusTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java index 4a27a0a88..edbaf1f97 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java @@ -44,7 +44,7 @@ /** * Process instance suspend tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessInstanceSuspendTest extends BaseTestClass { private String baseTestHDFSDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java index 6a12fc82a..0660c74f9 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLateRerunTest.java @@ -52,7 +52,7 @@ /** * Process late data test. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class ProcessLateRerunTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); private OozieClient cluster1OC = serverOC.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java index 8422796e2..11fe138dd 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathLoadTest.java @@ -55,7 +55,7 @@ /** * Tests with process lib folder with workflow.xml. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class ProcessLibPathLoadTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java index 4196d99cd..6e195866c 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessLibPathTest.java @@ -44,7 +44,7 @@ /** * Tests with process lib folder detached from workflow.xml. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessLibPathTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessSLATest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessSLATest.java index f4c9b3096..514fd68f0 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessSLATest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessSLATest.java @@ -39,7 +39,7 @@ /** * Process SLA tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessSLATest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java index dbb45a6f6..40061b55c 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java @@ -43,7 +43,7 @@ /** * Tests related to update feature. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessUpdateTest extends BaseTestClass { private OozieClient clusterOC = serverOC.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java index 25cf16f62..297ef5d2a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java @@ -45,7 +45,7 @@ /** * Touch feature test both via server and prism. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class TouchAPIPrismAndServerTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private OozieClient clusterOC = serverOC.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ValidateAPIPrismAndServerTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ValidateAPIPrismAndServerTest.java index ca612b809..1368519e7 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ValidateAPIPrismAndServerTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ValidateAPIPrismAndServerTest.java @@ -37,7 +37,7 @@ /** * Validate API is exposed both via server and prism. */ -@Test(groups = {"singleCluster"}) +@Test(groups = { "distributed", "embedded", "sanity" }) public class ValidateAPIPrismAndServerTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private String baseTestHDFSDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/EntitiesPatternSearchTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/EntitiesPatternSearchTest.java index 461b733a9..8f6b0e46b 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/EntitiesPatternSearchTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/EntitiesPatternSearchTest.java @@ -49,7 +49,7 @@ /** * Testing the pattern search of entities. Falcon-914 */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class EntitiesPatternSearchTest extends BaseTestClass { private static final Logger LOGGER = Logger.getLogger(EntitiesPatternSearchTest.class); private String baseTestHDFSDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/ListEntitiesTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/ListEntitiesTest.java index 011d02d15..e06ff9091 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/ListEntitiesTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/entity/ListEntitiesTest.java @@ -51,7 +51,7 @@ /** * Testing the list entities API. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class ListEntitiesTest extends BaseTestClass { private static final Logger LOGGER = Logger.getLogger(ListEntitiesTest.class); private String baseTestHDFSDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/EntityLineageTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/EntityLineageTest.java index 52cb1980d..29fc87150 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/EntityLineageTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/EntityLineageTest.java @@ -50,7 +50,7 @@ /** * Test Suite for Entity lineage. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class EntityLineageTest extends BaseTestClass { private String baseTestDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java index b3ca4f0fd..4f865942c 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java @@ -59,7 +59,7 @@ * expected instance statuses which are being compared with actual result of -list request * with different parameters in different order, variation, etc. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ListFeedInstancesTest extends BaseTestClass { private static final Logger LOGGER = Logger.getLogger(ListFeedInstancesTest.class); private OozieClient cluster2OC = serverOC.get(1); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListProcessInstancesTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListProcessInstancesTest.java index a4022a8e0..45cc9bedc 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListProcessInstancesTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListProcessInstancesTest.java @@ -47,7 +47,7 @@ /** * Test list instances api for process. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ListProcessInstancesTest extends BaseTestClass { private static final Logger LOGGER = Logger.getLogger(ListProcessInstancesTest.class); private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ProcessPipelineTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ProcessPipelineTest.java index bd4b9c2a5..a0b6e4fec 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ProcessPipelineTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ProcessPipelineTest.java @@ -48,7 +48,7 @@ /** * Submitting and listing processes with different pipeline names. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class ProcessPipelineTest extends BaseTestClass{ private static final Logger LOGGER = Logger.getLogger(ProcessPipelineTest.class); private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java index 4dc776f21..36cf70498 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/EntityDryRunTest.java @@ -44,7 +44,7 @@ /** * test cases for https://issues.apache.org/jira/browse/FALCON-353. */ -@Test(groups = "embedded") +@Test(groups = { "prism", "distributed", "embedded" }) public class EntityDryRunTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/OptionalInputTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/OptionalInputTest.java index 944c67f0a..61855de1b 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/OptionalInputTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/OptionalInputTest.java @@ -44,7 +44,7 @@ /** * Tests with optional input. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class OptionalInputTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismConcurrentRequestTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismConcurrentRequestTest.java index 515e97ecd..9a96b6a92 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismConcurrentRequestTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismConcurrentRequestTest.java @@ -42,7 +42,7 @@ /** * Tests with concurrent request to prism. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class PrismConcurrentRequestTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java index c7d9d2dc5..405725db0 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedReplicationUpdateTest.java @@ -49,7 +49,7 @@ /** * Update replication feed tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class PrismFeedReplicationUpdateTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedScheduleTest.java index a7bef6d80..137ef6f04 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedScheduleTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedScheduleTest.java @@ -38,7 +38,7 @@ /** * Schedule feed via prism tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class PrismFeedScheduleTest extends BaseTestClass { private OozieClient cluster1OC = serverOC.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java index dee8e3353..98d6f4e16 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSnSTest.java @@ -44,6 +44,7 @@ /** * Submit and schedule feed via prism tests. */ +@Test(groups = "distributed") public class PrismFeedSnSTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSuspendTest.java index c02a28fff..086a01493 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSuspendTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedSuspendTest.java @@ -38,6 +38,7 @@ /** * Suspend feed via prism tests. */ +@Test(groups = "distributed") public class PrismFeedSuspendTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java index ed4826554..a5220e3d5 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismFeedUpdateTest.java @@ -53,7 +53,7 @@ /** * Update feed via prism tests. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class PrismFeedUpdateTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java index 2dd602a50..e571730d3 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessScheduleTest.java @@ -45,6 +45,7 @@ /** * Schedule process via prism tests. */ +@Test(groups = "distributed") public class PrismProcessScheduleTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSuspendTest.java index e38cc67ed..752d18ca3 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSuspendTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/PrismProcessSuspendTest.java @@ -38,6 +38,7 @@ /** * Suspend process tests. */ +@Test(groups = "distributed") public class PrismProcessSuspendTest extends BaseTestClass { private ColoHelper cluster1 = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/ProcessPartitionExpVariableTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/ProcessPartitionExpVariableTest.java index 30b20c54c..0b3f16dfc 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/ProcessPartitionExpVariableTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/ProcessPartitionExpVariableTest.java @@ -46,7 +46,7 @@ /** * Tests with partitions as expression language variables. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class ProcessPartitionExpVariableTest extends BaseTestClass { private static final Logger LOGGER = Logger.getLogger(ProcessPartitionExpVariableTest.class); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java index c4fa214fe..df5269bde 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleKilledProcessTest.java @@ -40,7 +40,7 @@ /** * Tests with rescheduling killed process. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class RescheduleKilledProcessTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleProcessInFinalStatesTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleProcessInFinalStatesTest.java index 5031de487..a2b5d8dc2 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleProcessInFinalStatesTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RescheduleProcessInFinalStatesTest.java @@ -52,7 +52,7 @@ /** * Tests with rescheduling process in final states. */ -@Test(groups = "embedded", enabled = true) +@Test(groups = { "distributed", "embedded" }) public class RescheduleProcessInFinalStatesTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java index b6774334b..0ea9514ed 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java @@ -72,7 +72,7 @@ /** * Test with retention. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class RetentionTest extends BaseTestClass { private static final String TEST_FOLDERS = "/testFolders/"; private String baseTestHDFSDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/UpdateAtSpecificTimeTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/UpdateAtSpecificTimeTest.java index c6861caaa..f2eae6219 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/UpdateAtSpecificTimeTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/UpdateAtSpecificTimeTest.java @@ -61,6 +61,7 @@ /** * Test updating of feed with custom update time. */ +@Test(groups = "distributed") public class UpdateAtSpecificTimeTest extends BaseTestClass { private static final Logger LOGGER = Logger.getLogger(UpdateAtSpecificTimeTest.class); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/FeedInstanceDependencyTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/FeedInstanceDependencyTest.java index fe9f565c8..e7ddcd1a5 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/FeedInstanceDependencyTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/FeedInstanceDependencyTest.java @@ -44,7 +44,7 @@ /** * Test Suite for feed InstanceDependency corresponding to FALCON-1039. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class FeedInstanceDependencyTest extends BaseTestClass { private String baseTestDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/PipelineInstanceDependencyTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/PipelineInstanceDependencyTest.java index 81d5fcc7e..a4940487f 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/PipelineInstanceDependencyTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/PipelineInstanceDependencyTest.java @@ -66,7 +66,7 @@ /** * Test for instance dependency endpoint. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded", "sanity" }) public class PipelineInstanceDependencyTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/ProcessInstanceDependencyTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/ProcessInstanceDependencyTest.java index e93cd3665..b89060d40 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/ProcessInstanceDependencyTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/ProcessInstanceDependencyTest.java @@ -44,7 +44,7 @@ /** * Test Suite for process InstanceDependency corresponding to FALCON-1039. */ -@Test(groups = "embedded") +@Test(groups = { "distributed", "embedded" }) public class ProcessInstanceDependencyTest extends BaseTestClass { private String baseTestDir = cleanAndGetTestDir(); diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/TriageAPISingleColoTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/TriageAPISingleColoTest.java index 70d1e17d4..514b1c1fb 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/TriageAPISingleColoTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/triage/TriageAPISingleColoTest.java @@ -51,7 +51,7 @@ /** * Test Class for Testing the Triage API on a single colo corresponding to FALCON-1377. */ -@Test(groups = {"singleCluster"}) +@Test(groups = { "distributed", "embedded", "sanity" }) public class TriageAPISingleColoTest extends BaseTestClass { private ColoHelper cluster = servers.get(0); private OozieClient clusterOC = serverOC.get(0); From 97a155f8efeccd68870a8f80f95ae3b059009d55 Mon Sep 17 00:00:00 2001 From: Pragya Date: Mon, 29 Feb 2016 08:45:02 +0000 Subject: [PATCH 07/29] Review comments addressed --- .../core/util/NativeInstanceUtil.java | 18 +++++++++++------- .../NativeSchedulerAPITest.java | 6 +++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java index 97f01ab54..e809ee74d 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java @@ -32,7 +32,9 @@ import java.net.URISyntaxException; /** - * util functions related to instanceTest. + * Util functions related to native instance test. + * In case of entity scheduled using native scheduler, we cannot use oozie coordinator to retrieve relevant information + * This util retrieves instance related information using falcon APIs */ public final class NativeInstanceUtil { @@ -46,6 +48,7 @@ private NativeInstanceUtil() { /** * Waits till instance of specific entity will be created during timeout. * Timeout is common for most of usual test cases. + * Using status API to retrieve instance information. * * @param cluster ColoHelper - colo on which API to be executed * @param entity definition of entity which describes job @@ -55,14 +58,14 @@ private NativeInstanceUtil() { public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity, String startTime, String endTime) throws InterruptedException, IOException, AuthenticationException, URISyntaxException { - int sleep = INSTANCES_CREATED_TIMEOUT; - waitTillInstancesAreCreated(cluster, entity, startTime, endTime, sleep); + waitTillInstancesAreCreated(cluster, entity, startTime, endTime, INSTANCES_CREATED_TIMEOUT); } /** * Waits till instances of specific job will be created during specific time. * Use this method directly in unusual test cases where timeouts are different from trivial. * In other cases use waitTillInstancesAreCreated(ColoHelper,Entity,String,String) + * Using status API to retrieve instance information. * * @param cluster ColoHelper - colo on which API to be executed * @param entity definition of entity which describes job @@ -84,12 +87,13 @@ public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity break; } LOGGER.info(type + " " + entityName + " still doesn't have instance created"); - TimeUtil.sleepSeconds(60); + TimeUtil.sleepSeconds(10); } } /** * Waits till given instance of process/feed reach expected state during specific time. + * Using status API to retrieve instance information. * * @param cluster ColoHelper - colo on which API to be executed * @param entity definition of entity which describes job @@ -105,8 +109,8 @@ public static void waitTillInstanceReachState(ColoHelper cluster, Entity entity, } /** - * Waits till given instance of process/feed reach expected state during - * specific time. + * Waits till given instance of process/feed reach expected state during specific time. + * Using status API to retrieve instance information. * * @param cluster ColoHelper - colo on which API to be executed * @param entity definition of entity which describes job @@ -131,7 +135,7 @@ public static void waitTillInstanceReachState(ColoHelper cluster, Entity entity, for (int i = 0; i < maxTries; i++) { InstancesResult statusResult = cluster.getProcessHelper().getProcessInstanceStatus(entityName, params); if (statusResult.getInstances() != null) { - if (statusResult.getInstances()[0].getStatus().name().equals(expectedStatus.toString())) { + if (statusResult.getInstances()[0].getStatus().name() == expectedStatus.name()) { return; } } diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java index f8a6eb328..72bf503a5 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java @@ -106,7 +106,7 @@ public void tearDown() { /*Suspend and resume entity and check that it is successful. */ - @Test(enabled = true) + @Test public void entitySuspendResumeTest() throws Exception { //Suspend entity via prism AssertUtil.assertSucceeded(prism.getProcessHelper().suspend(processMerlin.toString())); @@ -139,7 +139,7 @@ public void entitySuspendResumeTest() throws Exception { /*Test for entity definition, list, dependency, update and summary API and check that it is successful. */ - @Test(enabled = true) + @Test public void entityDefinitionListDependencyUpdateSummaryTest() throws Exception { // Entity Definition String processDef = prism.getProcessHelper().getEntityDefinition(processMerlin.toString()).getMessage(); @@ -178,7 +178,7 @@ public void entityDefinitionListDependencyUpdateSummaryTest() throws Exception { /*Test for instance dependency, list, params, logs, running and triage API and check that it is successful. */ - @Test(enabled = true) + @Test public void instanceAPITest() throws Exception { // Instance dependency NativeInstanceUtil.waitTillInstanceReachState(prism, processMerlin, startTime, CoordinatorAction.Status.RUNNING, From b92d3f28a58a0ab568f8ec6a54e6333f5ea69a17 Mon Sep 17 00:00:00 2001 From: Pragya Date: Mon, 29 Feb 2016 09:23:09 +0000 Subject: [PATCH 08/29] Review comments addressed --- .../java/org/apache/falcon/regression/InstanceSummaryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java index 726a6ed64..df1716f3b 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/InstanceSummaryTest.java @@ -174,7 +174,7 @@ public void testSummarySingleClusterProcess() * Adjust multi-cluster process. Submit and schedule it. Get its instances summary. */ @Test(enabled = true, timeOut = 1200000) - public void testSummaryMultiClusterProcess() throws JAXBException, + public void testSummaryMultiClusterProcess() throws JAXBException, ParseException, IOException, URISyntaxException, AuthenticationException, InterruptedException { processBundle.setProcessValidity(startTime, endTime); From 9a882dfcbd95a2fac62a8f9145273881bea8e8eb Mon Sep 17 00:00:00 2001 From: Pragya Date: Mon, 29 Feb 2016 10:03:16 +0000 Subject: [PATCH 09/29] Modifying totalMinutesToWait in NativeInstancEUtil to wait for defined time --- .../apache/falcon/regression/core/util/NativeInstanceUtil.java | 1 + 1 file changed, 1 insertion(+) diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java index e809ee74d..b3f274536 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java @@ -80,6 +80,7 @@ public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity EntityType type = entity.getEntityType(); String params = "?start=" + startTime; params += (endTime.isEmpty() ? "" : "&end=" + endTime); + totalMinutesToWait = (totalMinutesToWait*60)/10; for (int sleepCount = 0; sleepCount < totalMinutesToWait; sleepCount++) { InstancesResult statusResult = cluster.getProcessHelper().getProcessInstanceStatus(entityName, params); From d244279d22e386fa9519174192684a393192ca17 Mon Sep 17 00:00:00 2001 From: Pragya Date: Tue, 1 Mar 2016 06:31:59 +0000 Subject: [PATCH 10/29] Modifying NativeInstanceUtil.waitTillInstancesAreCreated based on review comments --- .../falcon/regression/core/util/NativeInstanceUtil.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java index b3f274536..7b392e40c 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java @@ -80,9 +80,9 @@ public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity EntityType type = entity.getEntityType(); String params = "?start=" + startTime; params += (endTime.isEmpty() ? "" : "&end=" + endTime); - totalMinutesToWait = (totalMinutesToWait*60)/10; - for (int sleepCount = 0; sleepCount < totalMinutesToWait; sleepCount++) { + int totalSecondsToWait = totalMinutesToWait * 60; + for (int sleepSeconds = 0; sleepSeconds < totalSecondsToWait; sleepSeconds = sleepSeconds+10) { InstancesResult statusResult = cluster.getProcessHelper().getProcessInstanceStatus(entityName, params); if (statusResult.getInstances() != null) { break; @@ -90,6 +90,7 @@ public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity LOGGER.info(type + " " + entityName + " still doesn't have instance created"); TimeUtil.sleepSeconds(10); } + } /** From cb335be5f66f90a6b2ddb3c1703e0cd6ea3948ed Mon Sep 17 00:00:00 2001 From: Pragya Date: Tue, 1 Mar 2016 06:39:58 +0000 Subject: [PATCH 11/29] Modifying CHANGES.txt --- falcon-regression/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index e3f72640c..0fff00748 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -127,6 +127,8 @@ Trunk (Unreleased) via Samarth Gupta) IMPROVEMENTS + FALCON-1841 Grouping test in falcon for running nightly regression(Pragya Mittal) + FALCON-1819 Improve test class entity cleanup logic (Paul Isaychuk) FALCON-1697 Stabilization of scenarios which are based on instances lifecycle(Paul Isaychuk via Ajay Yadava) From 8b954484758a44702059829128d6eea58b8f33bb Mon Sep 17 00:00:00 2001 From: Pragya Date: Tue, 1 Mar 2016 07:31:21 +0000 Subject: [PATCH 12/29] Adding doc for NativeInstanceUtil.waitTillInstancesAreCreated --- .../falcon/regression/core/util/NativeInstanceUtil.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java index 7b392e40c..ffcf240c5 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java @@ -81,7 +81,11 @@ public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity String params = "?start=" + startTime; params += (endTime.isEmpty() ? "" : "&end=" + endTime); + // totalSecondsToWait (totalMinutesToWait in seconds) int totalSecondsToWait = totalMinutesToWait * 60; + + //Incrementing sleepSeconds by 10 so that looping happens (totalMinutesToWait*60)/10 times + //since TimeUtil.sleepSeconds is 10 for (int sleepSeconds = 0; sleepSeconds < totalSecondsToWait; sleepSeconds = sleepSeconds+10) { InstancesResult statusResult = cluster.getProcessHelper().getProcessInstanceStatus(entityName, params); if (statusResult.getInstances() != null) { From 9009d3979dd21fb040727c8e113dca8a17b1a345 Mon Sep 17 00:00:00 2001 From: Pragya Date: Tue, 1 Mar 2016 08:42:05 +0000 Subject: [PATCH 13/29] Adding assertion --- .../falcon/regression/core/util/NativeInstanceUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java index ffcf240c5..62b0f3f1b 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java @@ -89,12 +89,12 @@ public static void waitTillInstancesAreCreated(ColoHelper cluster, Entity entity for (int sleepSeconds = 0; sleepSeconds < totalSecondsToWait; sleepSeconds = sleepSeconds+10) { InstancesResult statusResult = cluster.getProcessHelper().getProcessInstanceStatus(entityName, params); if (statusResult.getInstances() != null) { - break; + return; } LOGGER.info(type + " " + entityName + " still doesn't have instance created"); TimeUtil.sleepSeconds(10); } - + Assert.fail("Instances not created"); } /** From 4fd3cf80eea4723a5a63da7193d9f83f20de71df Mon Sep 17 00:00:00 2001 From: Pragya Date: Thu, 3 Mar 2016 12:34:10 +0000 Subject: [PATCH 14/29] Making waitTillInstanceReachState generic --- .../falcon/regression/core/util/NativeInstanceUtil.java | 6 ++---- .../regression/nativeScheduler/NativeSchedulerAPITest.java | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java index 62b0f3f1b..aaf739071 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java @@ -134,11 +134,9 @@ public static void waitTillInstanceReachState(ColoHelper cluster, Entity entity, String endTime=getNextInstanceTime(instanceTime, frequency); String params = "?start=" + instanceTime + "&end=" + endTime; - int maxTries = 50; int totalSleepTime = totalMinutesToWait * 60; - int sleepTime = totalSleepTime / maxTries; - LOGGER.info(String.format("Sleep for %d seconds", sleepTime)); - for (int i = 0; i < maxTries; i++) { + int sleepTime = 10; + for (int i = 0; i < totalSleepTime; i = i+10) { InstancesResult statusResult = cluster.getProcessHelper().getProcessInstanceStatus(entityName, params); if (statusResult.getInstances() != null) { if (statusResult.getInstances()[0].getStatus().name() == expectedStatus.name()) { diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java index 72bf503a5..55ea2fb6b 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/nativeScheduler/NativeSchedulerAPITest.java @@ -271,7 +271,8 @@ public void instanceKillRerunAPITest() throws Exception { processMerlin.getFrequency()); //Instance Kill - InstancesResult instancesResult = prism.getProcessHelper().getProcessInstanceKill(processMerlin.getName(), range); + InstancesResult instancesResult = prism.getProcessHelper().getProcessInstanceKill(processMerlin.getName(), + range); InstanceUtil.validateResponse(instancesResult, 5, 0, 0, 0, 5); // Instance rerun @@ -281,7 +282,7 @@ public void instanceKillRerunAPITest() throws Exception { processMerlin.getFrequency()); instancesResult = prism.getProcessHelper().getProcessInstanceStatus(processMerlin.getName(), range); - InstanceUtil.validateResponse(instancesResult, 5, 2, 0, 0, 2); + InstanceUtil.validateResponse(instancesResult, 5, 2, 0, 0, 3); } From c8004058224f9b66fd5de60317e44702954afba1 Mon Sep 17 00:00:00 2001 From: Pragya Date: Fri, 4 Mar 2016 11:44:29 +0000 Subject: [PATCH 15/29] Addressed review comments --- .../apache/falcon/regression/core/util/NativeInstanceUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java index aaf739071..3add7a08f 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/NativeInstanceUtil.java @@ -136,7 +136,7 @@ public static void waitTillInstanceReachState(ColoHelper cluster, Entity entity, int totalSleepTime = totalMinutesToWait * 60; int sleepTime = 10; - for (int i = 0; i < totalSleepTime; i = i+10) { + for (int i = 0; i < totalSleepTime; i = i+sleepTime) { InstancesResult statusResult = cluster.getProcessHelper().getProcessInstanceStatus(entityName, params); if (statusResult.getInstances() != null) { if (statusResult.getInstances()[0].getStatus().name() == expectedStatus.name()) { From a7baab5e5f9b1bc05fcecc72b6a507bae72d6579 Mon Sep 17 00:00:00 2001 From: Pragya Date: Mon, 7 Mar 2016 06:04:24 +0000 Subject: [PATCH 16/29] Modified Changes.txt --- falcon-regression/CHANGES.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index e3f72640c..bc76009a5 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -5,6 +5,8 @@ Trunk (Unreleased) INCOMPATIBLE CHANGES NEW FEATURES + FALCON-1839 Test case for APIs for entities scheduled on native scheduler (Pragya Mittal) + FALCON-1566 Add test for SLA monitoring API (Pragya Mittal) FALCON-1567 Test case for Lifecycle feature (Pragya Mittal) @@ -127,6 +129,8 @@ Trunk (Unreleased) via Samarth Gupta) IMPROVEMENTS + FALCON-1841 Grouping test in falcon for running nightly regression(Pragya Mittal) + FALCON-1819 Improve test class entity cleanup logic (Paul Isaychuk) FALCON-1697 Stabilization of scenarios which are based on instances lifecycle(Paul Isaychuk via Ajay Yadava) From 93a7253de8026fa2f335775997a8f34d41d28aff Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Mon, 20 Nov 2017 20:10:41 +0530 Subject: [PATCH 17/29] Initial archival patch (Checkstyles, docs and test cases not handled) --- .../java/org/apache/falcon/LifeCycle.java | 3 +- .../src/main/java/org/apache/falcon/Tag.java | 3 +- common-types/src/main/resources/feed-0.1.xsd | 19 ++ .../org/apache/falcon/entity/FeedHelper.java | 47 ++- .../falcon/lifecycle/FeedLifecycleStage.java | 3 +- .../lifecycle/archival/AgeBasedArchival.java | 35 +++ .../lifecycle/archival/ArchivalPolicy.java | 54 ++++ common/src/main/resources/startup.properties | 7 +- .../test/resources/config/feed/feed-0.3.xml | 5 + .../archival/AgeBasedArchivalBuilder.java | 53 ++++ .../AgeBasedArchivalCoordinatorBuilder.java | 290 ++++++++++++++++++ .../AgeBasedArchivalWorkflowBuilder.java | 206 +++++++++++++ .../engine/oozie/utils/OozieBuilderUtils.java | 24 ++ .../resources/action/feed/archival-action.xml | 64 ++++ .../main/resources/action/post-process.xml | 98 ++++++ .../src/main/resources/action/pre-process.xml | 54 ++++ .../falcon/oozie/feed/FeedBundleBuilder.java | 2 +- 17 files changed, 945 insertions(+), 22 deletions(-) create mode 100644 common/src/main/java/org/apache/falcon/lifecycle/archival/AgeBasedArchival.java create mode 100644 common/src/main/java/org/apache/falcon/lifecycle/archival/ArchivalPolicy.java create mode 100644 lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalBuilder.java create mode 100644 lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java create mode 100644 lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java create mode 100644 lifecycle/src/main/resources/action/feed/archival-action.xml create mode 100644 lifecycle/src/main/resources/action/post-process.xml create mode 100644 lifecycle/src/main/resources/action/pre-process.xml diff --git a/common-types/src/main/java/org/apache/falcon/LifeCycle.java b/common-types/src/main/java/org/apache/falcon/LifeCycle.java index 0ecddd1ea..766a9612a 100644 --- a/common-types/src/main/java/org/apache/falcon/LifeCycle.java +++ b/common-types/src/main/java/org/apache/falcon/LifeCycle.java @@ -27,7 +27,8 @@ public enum LifeCycle { EVICTION(Tag.RETENTION), REPLICATION(Tag.REPLICATION), IMPORT(Tag.IMPORT), - EXPORT(Tag.EXPORT); + EXPORT(Tag.EXPORT), + ARCHIVAL(Tag.ARCHIVAL); private final Tag tag; diff --git a/common-types/src/main/java/org/apache/falcon/Tag.java b/common-types/src/main/java/org/apache/falcon/Tag.java index 3c2f25e20..d2efaae9a 100644 --- a/common-types/src/main/java/org/apache/falcon/Tag.java +++ b/common-types/src/main/java/org/apache/falcon/Tag.java @@ -28,7 +28,8 @@ public enum Tag { RETENTION(EntityType.FEED), REPLICATION(EntityType.FEED), IMPORT(EntityType.FEED), - EXPORT(EntityType.FEED); + EXPORT(EntityType.FEED), + ARCHIVAL(EntityType.FEED); private final EntityType entityType; diff --git a/common-types/src/main/resources/feed-0.1.xsd b/common-types/src/main/resources/feed-0.1.xsd index cbc97b970..3c8a84a8d 100644 --- a/common-types/src/main/resources/feed-0.1.xsd +++ b/common-types/src/main/resources/feed-0.1.xsd @@ -364,6 +364,7 @@ + @@ -575,4 +576,22 @@ + + + + + Archival stage is the new way to define archival for a feed using feed lifecycle feature. Archival + has a configurable policy which does the validation and the real execution through workflow engine. + This method of specifying archival gives you more control for archival rather than defining archival + source and target in replication lifecycle stage of feed. + + + + + + + + + + diff --git a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java index 757359f9c..404385ddc 100644 --- a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java +++ b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java @@ -27,22 +27,7 @@ import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.Frequency; import org.apache.falcon.entity.v0.datasource.DatasourceType; -import org.apache.falcon.entity.v0.feed.CatalogTable; -import org.apache.falcon.entity.v0.feed.Cluster; -import org.apache.falcon.entity.v0.feed.ClusterType; -import org.apache.falcon.entity.v0.feed.ExtractMethod; -import org.apache.falcon.entity.v0.feed.Feed; -import org.apache.falcon.entity.v0.feed.FieldIncludeExclude; -import org.apache.falcon.entity.v0.feed.Lifecycle; -import org.apache.falcon.entity.v0.feed.Load; -import org.apache.falcon.entity.v0.feed.Location; -import org.apache.falcon.entity.v0.feed.LocationType; -import org.apache.falcon.entity.v0.feed.Locations; -import org.apache.falcon.entity.v0.feed.MergeType; -import org.apache.falcon.entity.v0.feed.Property; -import org.apache.falcon.entity.v0.feed.RetentionStage; -import org.apache.falcon.entity.v0.feed.Sla; -import org.apache.falcon.entity.v0.feed.Validity; +import org.apache.falcon.entity.v0.feed.*; import org.apache.falcon.entity.v0.process.Input; import org.apache.falcon.entity.v0.process.Output; import org.apache.falcon.entity.v0.process.Process; @@ -408,6 +393,20 @@ public static RetentionStage getRetentionStage(Feed feed, String clusterName) th return null; } + public static ArchivalStage getArchivalStage(Feed feed, String clusterName) throws FalconException { + if (isLifecycleEnabled(feed, clusterName)) { + Lifecycle globalLifecycle = feed.getLifecycle(); + Lifecycle clusterLifecycle = getCluster(feed, clusterName).getLifecycle(); + + if (clusterLifecycle != null && clusterLifecycle.getArchivalStage() != null) { + return clusterLifecycle.getArchivalStage(); + } else if (globalLifecycle != null) { + return globalLifecycle.getArchivalStage(); + } + } + return null; + } + public static Date getFeedValidityStart(Feed feed, String clusterName) throws FalconException { org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, clusterName); if (feedCluster != null) { @@ -441,6 +440,11 @@ public static List getPolicies(Feed feed, String clusterName) throws Fal policy = StringUtils.isBlank(policy) ? FeedLifecycleStage.RETENTION.getDefaultPolicyName() : policy; result.add(policy); + + policy = getArchivalStage(feed, clusterName).getPolicy(); + policy = StringUtils.isBlank(policy) + ? FeedLifecycleStage.ARCHIVAL.getDefaultPolicyName() : policy; + result.add(policy); } return result; } @@ -1295,4 +1299,15 @@ public static List getListing(Feed feed, String clusterName, return storage.getListing(feed, clusterName, locationType, start, end); } + public static String getArchivalPath(Feed feed, String clusterName) throws FalconException { + String archivalPath = ""; + ArchivalStage archivalStage = getArchivalStage(feed, clusterName); + if (archivalStage != null) { + Location location = archivalStage.getLocation(); + if ((location != null) && (location.getPath() != null)) { + archivalPath = location.getPath(); + } + } + return archivalPath; + } } diff --git a/common/src/main/java/org/apache/falcon/lifecycle/FeedLifecycleStage.java b/common/src/main/java/org/apache/falcon/lifecycle/FeedLifecycleStage.java index 833ad0488..5b2775688 100644 --- a/common/src/main/java/org/apache/falcon/lifecycle/FeedLifecycleStage.java +++ b/common/src/main/java/org/apache/falcon/lifecycle/FeedLifecycleStage.java @@ -22,7 +22,8 @@ */ public enum FeedLifecycleStage { - RETENTION("AgeBasedDelete"); + RETENTION("AgeBasedDelete"), + ARCHIVAL("AgeBasedArchival"); private String defaultPolicyName; diff --git a/common/src/main/java/org/apache/falcon/lifecycle/archival/AgeBasedArchival.java b/common/src/main/java/org/apache/falcon/lifecycle/archival/AgeBasedArchival.java new file mode 100644 index 000000000..f14a2a0b1 --- /dev/null +++ b/common/src/main/java/org/apache/falcon/lifecycle/archival/AgeBasedArchival.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.falcon.lifecycle.archival; + +import org.apache.falcon.FalconException; +import org.apache.falcon.entity.v0.feed.Feed; + +/** + * Archival policy which archives all instances of instance time depending on the given frequency. + * It will create the workflow and coordinators for this policy. + */ +public class AgeBasedArchival extends ArchivalPolicy { + + @Override + public void validate(Feed feed, String clusterName) throws FalconException { + + } +} diff --git a/common/src/main/java/org/apache/falcon/lifecycle/archival/ArchivalPolicy.java b/common/src/main/java/org/apache/falcon/lifecycle/archival/ArchivalPolicy.java new file mode 100644 index 000000000..b1c7df5e7 --- /dev/null +++ b/common/src/main/java/org/apache/falcon/lifecycle/archival/ArchivalPolicy.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.lifecycle.archival; + +import org.apache.falcon.FalconException; +import org.apache.falcon.entity.v0.cluster.Cluster; +import org.apache.falcon.entity.v0.feed.Feed; +import org.apache.falcon.lifecycle.AbstractPolicyBuilderFactory; +import org.apache.falcon.lifecycle.FeedLifecycleStage; +import org.apache.falcon.lifecycle.LifecyclePolicy; +import org.apache.falcon.lifecycle.PolicyBuilder; +import org.apache.falcon.workflow.WorkflowEngineFactory; +import org.apache.hadoop.fs.Path; + +import java.util.Properties; + +/** + * All archival policies must implement this interface. + */ +public abstract class ArchivalPolicy implements LifecyclePolicy { + + @Override + public String getName() { + return this.getClass().getSimpleName(); + } + + @Override + public FeedLifecycleStage getStage() { + return FeedLifecycleStage.ARCHIVAL; + } + + @Override + public Properties build(Cluster cluster, Path buildPath, Feed feed) throws FalconException { + AbstractPolicyBuilderFactory factory = WorkflowEngineFactory.getLifecycleEngine(); + PolicyBuilder builder = factory.getPolicyBuilder(getName()); + return builder.build(cluster, buildPath, feed); + } +} diff --git a/common/src/main/resources/startup.properties b/common/src/main/resources/startup.properties index f89990577..7878403ea 100644 --- a/common/src/main/resources/startup.properties +++ b/common/src/main/resources/startup.properties @@ -69,9 +69,12 @@ # List of Lifecycle policies configured. -*.falcon.feed.lifecycle.policies=org.apache.falcon.lifecycle.retention.AgeBasedDelete +*.falcon.feed.lifecycle.policies=org.apache.falcon.lifecycle.retention.AgeBasedDelete,\ + org.apache.falcon.lifecycle.archival.AgeBasedArchival + # List of builders for the policies. -*.falcon.feed.lifecycle.policy.builders=org.apache.falcon.lifecycle.engine.oozie.retention.AgeBasedDeleteBuilder +*.falcon.feed.lifecycle.policy.builders=org.apache.falcon.lifecycle.engine.oozie.retention.AgeBasedDeleteBuilder,\ + org.apache.falcon.lifecycle.engine.oozie.archival.AgeBasedArchivalBuilder ##### Falcon Configuration Store Change listeners ##### *.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ org.apache.falcon.entity.ColoClusterRelation,\ diff --git a/common/src/test/resources/config/feed/feed-0.3.xml b/common/src/test/resources/config/feed/feed-0.3.xml index e6d3e01ac..360b1d436 100644 --- a/common/src/test/resources/config/feed/feed-0.3.xml +++ b/common/src/test/resources/config/feed/feed-0.3.xml @@ -79,5 +79,10 @@ + + false + default + + diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalBuilder.java new file mode 100644 index 000000000..0bab471af --- /dev/null +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalBuilder.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.lifecycle.engine.oozie.archival; + +import org.apache.falcon.FalconException; +import org.apache.falcon.entity.v0.cluster.Cluster; +import org.apache.falcon.entity.v0.feed.Feed; +import org.apache.falcon.lifecycle.PolicyBuilder; +import org.apache.falcon.lifecycle.archival.AgeBasedArchival; +import org.apache.hadoop.fs.Path; + +import java.util.Properties; + +public class AgeBasedArchivalBuilder implements PolicyBuilder { + + private static final String NAME = new AgeBasedArchival().getName(); + + @Override + public Properties build(Cluster cluster, Path buildPath, Feed feed) throws FalconException { + Properties wfProps = buildWorkflow(cluster, buildPath, feed); + return buildCoordinator(cluster, buildPath, feed, wfProps); + } + + @Override + public String getPolicyName() { + return NAME; + } + + public Properties buildCoordinator(Cluster cluster, Path buildPath, Feed feed, Properties wfProps) + throws FalconException { + return AgeBasedArchivalCoordinatorBuilder.build(cluster, buildPath, feed, wfProps); + } + + public Properties buildWorkflow(Cluster cluster, Path buildPath, Feed feed) throws FalconException { + return AgeBasedArchivalWorkflowBuilder.build(cluster, buildPath, feed); + } +} diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java new file mode 100644 index 000000000..d27a2dc23 --- /dev/null +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java @@ -0,0 +1,290 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.lifecycle.engine.oozie.archival; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.falcon.FalconException; +import org.apache.falcon.LifeCycle; +import org.apache.falcon.Tag; +import org.apache.falcon.entity.CatalogStorage; +import org.apache.falcon.entity.EntityUtil; +import org.apache.falcon.entity.FeedHelper; +import org.apache.falcon.entity.Storage; +import org.apache.falcon.entity.v0.Frequency; +import org.apache.falcon.entity.v0.SchemaHelper; +import org.apache.falcon.entity.v0.cluster.Cluster; +import org.apache.falcon.entity.v0.feed.Feed; +import org.apache.falcon.entity.v0.feed.LocationType; +import org.apache.falcon.entity.v0.feed.Property; +import org.apache.falcon.entity.v0.process.ExecutionType; +import org.apache.falcon.expression.ExpressionHelper; +import org.apache.falcon.lifecycle.engine.oozie.utils.OozieBuilderUtils; +import org.apache.falcon.oozie.coordinator.*; +import org.apache.falcon.util.RuntimeProperties; +import org.apache.falcon.workflow.WorkflowExecutionArgs; +import org.apache.falcon.workflow.WorkflowExecutionContext; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.json.XMLTokener.entity; + +public class AgeBasedArchivalCoordinatorBuilder { + + private AgeBasedArchivalCoordinatorBuilder() { + + } + + private static final Logger LOG = LoggerFactory.getLogger(AgeBasedArchivalCoordinatorBuilder.class); + private static final int THIRTY_MINUTES = 30 * 60 * 1000; + private static final String PARALLEL = "parallel"; + private static final String TIMEOUT = "timeout"; + private static final String ORDER = "order"; + + /** + * Builds the coordinator app. + * @param cluster - cluster to schedule retention on. + * @param basePath - Base path to marshal coordinator app. + * @param feed - feed for which retention is to be scheduled. + * @param wfProp - properties passed from workflow to coordinator e.g. ENTITY_PATH + * @return - Properties from creating the coordinator application to be used by Bundle. + * @throws FalconException + */ + public static Properties build(Cluster cluster, Path basePath, Feed feed, Properties wfProp) + throws FalconException { + + org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName()); + + // workflow is serialized to a specific dir + Path coordPath = new Path(basePath, Tag.ARCHIVAL.name() + "/" + feedCluster.getName()); + + COORDINATORAPP coord = new COORDINATORAPP(); + + String coordName = EntityUtil.getWorkflowName(LifeCycle.EVICTION.getTag(), feed).toString(); + + long replicationDelayInMillis = getReplicationDelayInMillis(feed, cluster); + Date sourceStartDate = getStartDate(feed, cluster, replicationDelayInMillis); + Date sourceEndDate = getEndDate(feed, cluster); + + coord.setName(coordName); + coord.setFrequency("${coord:" + feed.getFrequency().toString() + "}"); + coord.setEnd(SchemaHelper.formatDateUTC(sourceEndDate)); + coord.setStart(SchemaHelper.formatDateUTC(sourceStartDate)); + coord.setTimezone(feed.getTimezone().getID()); + + if (replicationDelayInMillis > 0) { + long delayInMins = -1 * replicationDelayInMillis / (1000 * 60); + String elExp = "${now(0," + delayInMins + ")}"; + + coord.getInputEvents().getDataIn().get(0).getInstance().set(0, elExp); + coord.getOutputEvents().getDataOut().get(0).setInstance(elExp); + } + + setCoordControls(feed, coord); + + final Storage sourceStorage = FeedHelper.createReadOnlyStorage(cluster, feed); + initializeInputDataSet(feed, cluster, coord, sourceStorage); + + String archivalPath = FeedHelper.getArchivalPath(feed, cluster.getName()); + initializeOutputDataSet(feed, cluster, coord, archivalPath); + + ACTION replicationWorkflowAction = getReplicationWorkflowAction(feed, cluster, coordPath, coordName, sourceStorage); + coord.setAction(replicationWorkflowAction); + + Path marshalPath = OozieBuilderUtils.marshalCoordinator(cluster, coord, coordPath); + wfProp.putAll(OozieBuilderUtils.getProperties(marshalPath, coordName)); + + return wfProp; + + } + + + private static ACTION getReplicationWorkflowAction(Feed feed, Cluster cluster, Path buildPath, String coordName, + Storage sourceStorage) throws FalconException { + ACTION action = new ACTION(); + WORKFLOW workflow = new WORKFLOW(); + + workflow.setAppPath(OozieBuilderUtils.getStoragePath(String.valueOf(buildPath))); + Properties props = OozieBuilderUtils.createCoordDefaultConfiguration(coordName, feed); + + // Setting CLUSTER_NAME property to include source cluster + props.put(WorkflowExecutionArgs.CLUSTER_NAME.getName(), cluster.getName()); + props.put("srcClusterName", cluster.getName()); + props.put("srcClusterColo", cluster.getColo()); + + // the storage type is uniform across source and target feeds for replication + props.put("falconFeedStorageType", sourceStorage.getType().name()); + + String instancePaths = ""; + if (sourceStorage.getType() == Storage.TYPE.FILESYSTEM) { + String pathsWithPartitions = getPathsWithPartitions(feed, cluster); + instancePaths = pathsWithPartitions; + propagateFileSystemCopyProperties(pathsWithPartitions, props); + } + + propagateLateDataProperties(feed, instancePaths, sourceStorage.getType().name(), props); + // Add the custom properties set in feed. Else, dryrun won't catch any missing props. + props.putAll(EntityUtil.getEntityProperties(feed)); + workflow.setConfiguration(OozieBuilderUtils.getCoordinatorConfig(props)); + action.setWorkflow(workflow); + + return action; + } + + private static String getPathsWithPartitions(Feed feed, Cluster cluster) throws FalconException { + String srcPart = FeedHelper.normalizePartitionExpression( + FeedHelper.getCluster(feed, cluster.getName()).getPartition()); + srcPart = FeedHelper.evaluateClusterExp(cluster, srcPart); + + StringBuilder pathsWithPartitions = new StringBuilder(); + pathsWithPartitions.append("${coord:dataIn('input')}/") + .append(FeedHelper.normalizePartitionExpression(srcPart)); + + String parts = pathsWithPartitions.toString().replaceAll("//+", "/"); + parts = StringUtils.stripEnd(parts, "/"); + return parts; + } + + private static void propagateLateDataProperties(Feed feed, String instancePaths, + String falconFeedStorageType, Properties props) { + // todo these pairs are the same but used in different context + // late data handler - should-record action + props.put(WorkflowExecutionArgs.INPUT_FEED_NAMES.getName(), feed.getName()); + props.put(WorkflowExecutionArgs.INPUT_FEED_PATHS.getName(), instancePaths); + props.put(WorkflowExecutionArgs.INPUT_NAMES.getName(), feed.getName()); + + // storage type for each corresponding feed - in this case only one feed is involved + // needed to compute usage based on storage type in LateDataHandler + props.put(WorkflowExecutionArgs.INPUT_STORAGE_TYPES.getName(), falconFeedStorageType); + + // falcon post processing + props.put(WorkflowExecutionArgs.OUTPUT_FEED_NAMES.getName(), feed.getName()); + props.put(WorkflowExecutionArgs.OUTPUT_NAMES.getName(), feed.getName()); + props.put(WorkflowExecutionArgs.OUTPUT_FEED_PATHS.getName(), "${coord:dataOut('output')}"); + } + + private static void propagateFileSystemCopyProperties(String paths, Properties props) throws FalconException { + props.put("sourceRelativePaths", paths); + + props.put("distcpSourcePaths", "${coord:dataIn('input')}"); + props.put("distcpTargetPaths", "${coord:dataOut('output')}"); + } + + private static void setCoordControls(Feed feed, COORDINATORAPP coord) throws FalconException { + // set controls + CONTROLS controls = new CONTROLS(); + + long frequencyInMillis = ExpressionHelper.get().evaluate(feed.getFrequency().toString(), Long.class); + long timeoutInMillis = frequencyInMillis * 6; + if (timeoutInMillis < THIRTY_MINUTES) { + timeoutInMillis = THIRTY_MINUTES; + } + + Properties props = EntityUtil.getEntityProperties(feed); + String timeout = props.getProperty(TIMEOUT); + if (timeout!=null) { + try{ + timeoutInMillis= ExpressionHelper.get().evaluate(timeout, Long.class); + } catch (Exception ignore) { + LOG.error("Unable to evaluate timeout:", ignore); + } + } + + String parallelProp = props.getProperty(PARALLEL); + int parallel = 1; + if (parallelProp != null) { + try { + parallel = Integer.parseInt(parallelProp); + } catch (NumberFormatException ignore) { + LOG.error("Unable to parse parallel:", ignore); + } + } + + String orderProp = props.getProperty(ORDER); + ExecutionType order = ExecutionType.FIFO; + if (orderProp != null) { + try { + order = ExecutionType.fromValue(orderProp); + } catch (IllegalArgumentException ignore) { + LOG.error("Unable to parse order:", ignore); + } + } + + controls.setTimeout(String.valueOf(timeoutInMillis / (1000 * 60))); + controls.setThrottle(String.valueOf(timeoutInMillis / frequencyInMillis * 2)); + controls.setConcurrency(String.valueOf(parallel)); + controls.setExecution(order.name()); + coord.setControls(controls); + } + + private static void initializeInputDataSet(Feed feed, Cluster cluster, COORDINATORAPP coord, Storage storage) throws FalconException { + SYNCDATASET inputDataset = (SYNCDATASET)coord.getDatasets().getDatasetOrAsyncDataset().get(0); + + String uriTemplate = storage.getUriTemplate(LocationType.DATA); + inputDataset.setUriTemplate(uriTemplate); + + setDatasetValues(feed, inputDataset, cluster); + + if (feed.getAvailabilityFlag() == null) { + inputDataset.setDoneFlag(""); + } else { + inputDataset.setDoneFlag(feed.getAvailabilityFlag()); + } + } + + private static void initializeOutputDataSet(Feed feed, Cluster cluster, COORDINATORAPP coord, + String targetPath) throws FalconException { + SYNCDATASET outputDataset = (SYNCDATASET)coord.getDatasets().getDatasetOrAsyncDataset().get(1); + outputDataset.setUriTemplate(targetPath); + + setDatasetValues(feed, outputDataset, cluster); + } + + + private static void setDatasetValues(Feed feed, SYNCDATASET dataset, Cluster cluster) { + dataset.setInitialInstance(SchemaHelper.formatDateUTC( + FeedHelper.getCluster(feed, cluster.getName()).getValidity().getStart())); + dataset.setTimezone(feed.getTimezone().getID()); + dataset.setFrequency("${coord:" + feed.getFrequency().toString() + "}"); + } + + private static long getReplicationDelayInMillis(Feed feed, Cluster srcCluster) throws FalconException { + Frequency replicationDelay = FeedHelper.getCluster(feed, srcCluster.getName()).getDelay(); + long delayInMillis=0; + if (replicationDelay != null) { + delayInMillis = ExpressionHelper.get().evaluate( + replicationDelay.toString(), Long.class); + } + + return delayInMillis; + } + + private static Date getStartDate(Feed feed, Cluster cluster, long replicationDelayInMillis) { + Date startDate = FeedHelper.getCluster(feed, cluster.getName()).getValidity().getStart(); + return replicationDelayInMillis == 0 ? startDate : new Date(startDate.getTime() + replicationDelayInMillis); + } + + private static Date getEndDate(Feed feed, Cluster cluster) { + return FeedHelper.getCluster(feed, cluster.getName()).getValidity().getEnd(); + } + +} diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java new file mode 100644 index 000000000..860167969 --- /dev/null +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.lifecycle.engine.oozie.archival; + +import org.apache.commons.lang3.StringUtils; +import org.apache.falcon.FalconException; +import org.apache.falcon.LifeCycle; +import org.apache.falcon.Tag; +import org.apache.falcon.entity.EntityUtil; +import org.apache.falcon.entity.FeedHelper; +import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.cluster.Cluster; +import org.apache.falcon.entity.v0.feed.Feed; +import org.apache.falcon.entity.v0.feed.Property; +import org.apache.falcon.lifecycle.engine.oozie.utils.OozieBuilderUtils; +import org.apache.falcon.oozie.workflow.*; +import org.apache.falcon.util.ReplicationDistCpOption; +import org.apache.falcon.util.RuntimeProperties; +import org.apache.falcon.workflow.WorkflowExecutionArgs; +import org.apache.falcon.workflow.WorkflowExecutionContext; +import org.apache.hadoop.fs.Path; + +import java.util.List; +import java.util.Properties; + +public class AgeBasedArchivalWorkflowBuilder { + private static final String ARCHIVAL_ACTION_TEMPLATE = "/action/feed/archival-action.xml"; + private static final String ARCHIVAL_ACTION_NAME = "archival"; + + private static final String ARCHIVAL_JOB_COUNTER = "job.counter"; + private static final String TDE_ENCRYPTION_ENABLED = "tdeEncryptionEnabled"; + private static final String MR_MAX_MAPS = "maxMaps"; + private static final String MR_MAP_BANDWIDTH = "mapBandwidth"; + + private AgeBasedArchivalWorkflowBuilder(){ + + } + + public static Properties build(Cluster cluster, Path basePath, Feed feed) throws FalconException { + Path buildPath = OozieBuilderUtils.getBuildPath(basePath, LifeCycle.ARCHIVAL.getTag()); + WORKFLOWAPP workflow = new WORKFLOWAPP(); + String wfName = EntityUtil.getWorkflowName(Tag.ARCHIVAL, feed).toString(); + + String start = ARCHIVAL_ACTION_NAME; + + //Add pre-processing + if (shouldPreProcess(feed)) { + ACTION action = OozieBuilderUtils.getPreProcessingAction(Tag.ARCHIVAL); + addTransition(action, ARCHIVAL_ACTION_NAME, OozieBuilderUtils.getFailAction()); + workflow.getDecisionOrForkOrJoin().add(action); + start = OozieBuilderUtils.PREPROCESS_ACTION_NAME; + } + + //Add replication + ACTION archival = OozieBuilderUtils.unmarshalAction(ARCHIVAL_ACTION_TEMPLATE); + addAdditionalArchivalProperties(feed, archival); + enableCounters(feed, archival); + enableTDE(feed, archival); + addPostProcessing(workflow, archival); + OozieBuilderUtils.decorateWorkflow(workflow, wfName, start); + + OozieBuilderUtils.addLibExtensionsToWorkflow(cluster, workflow, Tag.ARCHIVAL, EntityType.FEED); + + OozieBuilderUtils.marshalWokflow(cluster, workflow, buildPath); + + + Properties props = OozieBuilderUtils.getProperties(buildPath, wfName); + props.putAll(OozieBuilderUtils.createDefaultConfiguration(cluster, feed, WorkflowExecutionContext.EntityOperations.REPLICATE)); + + props.putAll(getWorkflowProperties(feed, cluster)); + props.putAll(FeedHelper.getUserWorkflowProperties(LifeCycle.ARCHIVAL)); + + // Write out the config to config-default.xml + OozieBuilderUtils.marshalDefaultConfig(cluster, workflow, props, buildPath); + return props; + + } + + private static Properties getWorkflowProperties(Feed feed, Cluster cluster) throws FalconException { + Properties props = FeedHelper.getFeedProperties(feed); + if (props.getProperty(MR_MAX_MAPS) == null) { // set default if user has not overridden + props.put(MR_MAX_MAPS, getDefaultMaxMaps()); + } + if (props.getProperty(MR_MAP_BANDWIDTH) == null) { // set default if user has not overridden + props.put(MR_MAP_BANDWIDTH, getDefaultMapBandwidth()); + } + + if (feed.getAvailabilityFlag() == null) { + props.put("availabilityFlag", "NA"); + } else { + props.put("availabilityFlag", feed.getAvailabilityFlag()); + } + props.put(WorkflowExecutionArgs.DATASOURCE_NAME.getName(), "NA"); + + return props; + } + + private static String getDefaultMaxMaps() { + return RuntimeProperties.get().getProperty("falcon.replication.workflow.maxmaps", "5"); + } + + private static String getDefaultMapBandwidth() { + return RuntimeProperties.get().getProperty("falcon.replication.workflow.mapbandwidth", "100"); + } + + private static boolean shouldPreProcess(Feed feed) throws FalconException { + return !(EntityUtil.getLateProcess(feed) == null + || EntityUtil.getLateProcess(feed).getLateInputs() == null + || EntityUtil.getLateProcess(feed).getLateInputs().size() == 0); + } + + private static void addTransition(ACTION action, String ok, String fail) { + action.getOk().setTo(ok); + action.getError().setTo(fail); + } + + private static void addAdditionalArchivalProperties(Feed feed, ACTION archivalAction) { + List args = archivalAction.getJava().getArg(); + Properties props = EntityUtil.getEntityProperties(feed); + + for (ReplicationDistCpOption distcpOption : ReplicationDistCpOption.values()) { + String propertyValue = props.getProperty(distcpOption.getName()); + if (StringUtils.isNotEmpty(propertyValue)) { + args.add("-" + distcpOption.getName()); + args.add(propertyValue); + } + } + } + + private static ACTION enableCounters(Feed feed, ACTION action) throws FalconException { + if (isCounterEnabled(feed)) { + List args = action.getJava().getArg(); + args.add("-counterLogDir"); + args.add("${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}"); + } + return action; + } + + private static boolean isCounterEnabled(Feed feed) throws FalconException { + if (feed.getProperties() != null) { + List propertyList = feed.getProperties().getProperties(); + for (Property prop : propertyList) { + if (prop.getName().equals(ARCHIVAL_JOB_COUNTER) && "true" .equalsIgnoreCase(prop.getValue())) { + return true; + } + } + } + return false; + } + + private static ACTION enableTDE(Feed feed, ACTION action) throws FalconException { + if (isTDEEnabled(feed)) { + List args = action.getJava().getArg(); + args.add("-tdeEncryptionEnabled"); + args.add("true"); + } + return action; + } + + private static boolean isTDEEnabled(Feed feed) { + String tdeEncryptionEnabled = FeedHelper.getPropertyValue(feed, TDE_ENCRYPTION_ENABLED); + return "true" .equalsIgnoreCase(tdeEncryptionEnabled); + } + + private static void addPostProcessing(WORKFLOWAPP workflow, ACTION action) throws FalconException{ + if (!Boolean.parseBoolean(OozieBuilderUtils.ENABLE_POSTPROCESSING)){ + OozieBuilderUtils.addTransition(action, OozieBuilderUtils.OK_ACTION_NAME, + OozieBuilderUtils.FAIL_ACTION_NAME); + workflow.getDecisionOrForkOrJoin().add(action); + } else { + OozieBuilderUtils.addTransition(action, OozieBuilderUtils.SUCCESS_POSTPROCESS_ACTION_NAME, + OozieBuilderUtils.FAIL_POSTPROCESS_ACTION_NAME); + workflow.getDecisionOrForkOrJoin().add(action); + + //Add post-processing actions + ACTION success = OozieBuilderUtils.getSuccessPostProcessAction(); + OozieBuilderUtils.addTransition(success, OozieBuilderUtils.OK_ACTION_NAME, + OozieBuilderUtils.FAIL_ACTION_NAME); + workflow.getDecisionOrForkOrJoin().add(success); + + ACTION fail = OozieBuilderUtils.getFailPostProcessAction(); + OozieBuilderUtils.addTransition(fail, OozieBuilderUtils.FAIL_ACTION_NAME, + OozieBuilderUtils.FAIL_ACTION_NAME); + workflow.getDecisionOrForkOrJoin().add(fail); + } + } + +} + + diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/utils/OozieBuilderUtils.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/utils/OozieBuilderUtils.java index 7d51c9a71..5970c3cb8 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/utils/OozieBuilderUtils.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/utils/OozieBuilderUtils.java @@ -79,6 +79,7 @@ public final class OozieBuilderUtils { private static final Logger LOG = LoggerFactory.getLogger(OozieBuilderUtils.class); private static final String POSTPROCESS_TEMPLATE = "/action/post-process.xml"; + private static final String PREPROCESS_TEMPLATE = "/action/pre-process.xml"; public static final String HIVE_CREDENTIAL_NAME = "falconHiveAuth"; public static final String MR_QUEUE_NAME = "queueName"; @@ -97,6 +98,7 @@ public final class OozieBuilderUtils { public static final String FAIL_POSTPROCESS_ACTION_NAME = "failed-post-processing"; public static final String OK_ACTION_NAME = "end"; public static final String FAIL_ACTION_NAME = "fail"; + public static final String PREPROCESS_ACTION_NAME = "pre-processing"; public static final String ENTITY_PATH = "ENTITY_PATH"; @@ -557,4 +559,26 @@ public static CONFIGURATION getCoordinatorConfig(Properties props) { } return conf; } + + public static ACTION getPreProcessingAction(Tag tag) throws FalconException { + ACTION action = unmarshalAction(PREPROCESS_TEMPLATE); + decorateWithOozieRetries(action); + + List args = action.getJava().getArg(); + args.add("-out"); + if (tag == Tag.ARCHIVAL) { + args.add("${logDir}/latedata/${nominalTime}/${srcClusterName}"); + } else { + args.add("${logDir}/latedata/${nominalTime}"); + } + return action; + } + + public static String getFailAction(){ + if (!Boolean.parseBoolean(OozieBuilderUtils.ENABLE_POSTPROCESSING)){ + return FAIL_ACTION_NAME; + } else { + return FAIL_POSTPROCESS_ACTION_NAME; + } + } } diff --git a/lifecycle/src/main/resources/action/feed/archival-action.xml b/lifecycle/src/main/resources/action/feed/archival-action.xml new file mode 100644 index 000000000..cca4517cb --- /dev/null +++ b/lifecycle/src/main/resources/action/feed/archival-action.xml @@ -0,0 +1,64 @@ + + + + + ${jobTracker} + ${nameNode} + + + oozie.launcher.mapreduce.job.user.classpath.first + true + + + mapred.job.queue.name + ${queueName} + + + oozie.launcher.mapred.job.priority + ${jobPriority} + + + oozie.action.sharelib.for.java + distcp + + + oozie.launcher.oozie.libpath + ${wf:conf("falcon.libpath")} + + + org.apache.falcon.replication.FeedReplicator + -Dfalcon.include.path=${sourceRelativePaths} + -Dmapred.job.queue.name=${queueName} + -Dmapred.job.priority=${jobPriority} + -maxMaps + ${maxMaps} + -mapBandwidth + ${mapBandwidth} + -sourcePaths + ${distcpSourcePaths} + -targetPath + ${distcpTargetPaths} + -falconFeedStorageType + ${falconFeedStorageType} + -availabilityFlag + ${availabilityFlag == 'NA' ? "NA" : availabilityFlag} + + + + diff --git a/lifecycle/src/main/resources/action/post-process.xml b/lifecycle/src/main/resources/action/post-process.xml new file mode 100644 index 000000000..d649a0f80 --- /dev/null +++ b/lifecycle/src/main/resources/action/post-process.xml @@ -0,0 +1,98 @@ + + + + ${jobTracker} + ${nameNode} + + + mapred.job.queue.name + ${queueName} + + + oozie.launcher.mapred.job.priority + ${jobPriority} + + + oozie.launcher.oozie.libpath + ${wf:conf("falcon.libpath")} + + + org.apache.falcon.workflow.FalconPostProcessing + -cluster + ${cluster} + -entityType + ${entityType} + -entityName + ${entityName} + -nominalTime + ${nominalTime} + -operation + ${falconDataOperation} + -workflowId + ${wf:id()} + -runId + ${wf:run()} + -status + ${wf:lastErrorNode() == null ? 'SUCCEEDED' : 'FAILED'} + -timeStamp + ${timeStamp} + -brokerImplClass + ${brokerImplClass} + -brokerUrl + ${brokerUrl} + -userBrokerImplClass + ${userBrokerImplClass} + -userBrokerUrl + ${userBrokerUrl} + -userJMSNotificationEnabled + ${userJMSNotificationEnabled} + -systemJMSNotificationEnabled + ${systemJMSNotificationEnabled} + -brokerTTL + ${brokerTTL} + -feedNames + ${feedNames} + -feedInstancePaths + ${feedInstancePaths} + -logFile + ${logDir}/job-${nominalTime}/${wf:run()}/evicted-instancePaths.csv + -workflowEngineUrl + ${workflowEngineUrl} + -subflowId + ${wf:id()}${userWorkflowEngine == "oozie" ? "@user-action" : ""} + -userWorkflowEngine + ${userWorkflowEngine} + -userWorkflowName + ${userWorkflowName} + -userWorkflowVersion + ${userWorkflowVersion} + -logDir + ${logDir}/job-${nominalTime}/${srcClusterName == 'NA' ? '' : srcClusterName}/ + -workflowUser + ${wf:user()} + -falconInputFeeds + ${falconInputFeeds} + -falconInPaths + ${falconInPaths} + -datasource + ${datasource == 'NA' ? 'IGNORE' : datasource} + + + + diff --git a/lifecycle/src/main/resources/action/pre-process.xml b/lifecycle/src/main/resources/action/pre-process.xml new file mode 100644 index 000000000..fc4125c05 --- /dev/null +++ b/lifecycle/src/main/resources/action/pre-process.xml @@ -0,0 +1,54 @@ + + + + ${jobTracker} + ${nameNode} + + + mapred.job.queue.name + ${queueName} + + + oozie.launcher.mapred.job.priority + ${jobPriority} + + + + oozie.action.sharelib.for.java + hcatalog + + + oozie.launcher.oozie.libpath + ${wf:conf("falcon.libpath")} + + + org.apache.falcon.workflow.LateDataHandler + -out + ${logDir}/latedata/${nominalTime}/${srcClusterName} + -paths + ${falconInPaths} + -falconInputNames + ${falconInputNames} + -falconInputFeedStorageTypes + ${falconInputFeedStorageTypes} + + + + + diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java index c7584119b..f6b18af39 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java @@ -58,7 +58,7 @@ protected List buildCoords(Cluster cluster, Path buildPath) throws F props.add(appProps); } } - } else { + } else if (FeedHelper.getRetentionStage(this.entity, cluster.getName()) == null){ List evictionProps = OozieCoordinatorBuilder.get(entity, Tag.RETENTION).buildCoords(cluster, buildPath); if (evictionProps != null) { From 204532f38ff58093677d3080185f4c872c58cff6 Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Tue, 21 Nov 2017 12:50:59 +0530 Subject: [PATCH 18/29] Adding queue to archival workflow --- .../oozie/archival/AgeBasedArchivalWorkflowBuilder.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java index 860167969..ee45666a7 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java @@ -26,6 +26,7 @@ import org.apache.falcon.entity.FeedHelper; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.cluster.Cluster; +import org.apache.falcon.entity.v0.feed.ArchivalStage; import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.entity.v0.feed.Property; import org.apache.falcon.lifecycle.engine.oozie.utils.OozieBuilderUtils; @@ -86,6 +87,11 @@ public static Properties build(Cluster cluster, Path basePath, Feed feed) throws props.putAll(getWorkflowProperties(feed, cluster)); props.putAll(FeedHelper.getUserWorkflowProperties(LifeCycle.ARCHIVAL)); + ArchivalStage archivalStage = FeedHelper.getArchivalStage(feed, cluster.getName()); + if (StringUtils.isNotBlank(archivalStage.getQueue())) { + props.put(OozieBuilderUtils.MR_QUEUE_NAME, archivalStage.getQueue()); + } + // Write out the config to config-default.xml OozieBuilderUtils.marshalDefaultConfig(cluster, workflow, props, buildPath); return props; From 753f7cae270f43c55a8a6d9e3e5b1c446064d0e1 Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Tue, 21 Nov 2017 17:20:28 +0530 Subject: [PATCH 19/29] Adding validation fro archivalPolicy --- common-types/src/main/resources/feed-0.1.xsd | 4 +- .../org/apache/falcon/entity/FeedHelper.java | 2 +- .../entity/parser/FeedEntityParser.java | 46 ++++++++++++++++--- .../lifecycle/retention/AgeBasedDelete.java | 5 +- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/common-types/src/main/resources/feed-0.1.xsd b/common-types/src/main/resources/feed-0.1.xsd index 3c8a84a8d..42b07e7aa 100644 --- a/common-types/src/main/resources/feed-0.1.xsd +++ b/common-types/src/main/resources/feed-0.1.xsd @@ -363,8 +363,8 @@ - - + + diff --git a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java index 404385ddc..2e9a212fc 100644 --- a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java +++ b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java @@ -386,7 +386,7 @@ public static RetentionStage getRetentionStage(Feed feed, String clusterName) th if (clusterLifecycle != null && clusterLifecycle.getRetentionStage() != null) { return clusterLifecycle.getRetentionStage(); - } else if (globalLifecycle != null) { + } else if (globalLifecycle != null && globalLifecycle.getRetentionStage() != null) { return globalLifecycle.getRetentionStage(); } } diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java index 6b72174ac..93eba76de 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java @@ -33,18 +33,18 @@ import org.apache.falcon.entity.v0.EntityGraph; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.Frequency; -import org.apache.falcon.entity.v0.feed.ACL; -import org.apache.falcon.entity.v0.feed.Extract; -import org.apache.falcon.entity.v0.feed.ExtractMethod; -import org.apache.falcon.entity.v0.feed.Feed; -import org.apache.falcon.entity.v0.feed.Cluster; -import org.apache.falcon.entity.v0.feed.ClusterType; -import org.apache.falcon.entity.v0.feed.Location; import org.apache.falcon.entity.v0.feed.LocationType; import org.apache.falcon.entity.v0.feed.MergeType; import org.apache.falcon.entity.v0.feed.Properties; import org.apache.falcon.entity.v0.feed.Property; import org.apache.falcon.entity.v0.feed.Sla; +import org.apache.falcon.entity.v0.feed.Feed; +import org.apache.falcon.entity.v0.feed.Cluster; +import org.apache.falcon.entity.v0.feed.ClusterType; +import org.apache.falcon.entity.v0.feed.ACL; +import org.apache.falcon.entity.v0.feed.Location; +import org.apache.falcon.entity.v0.feed.Extract; +import org.apache.falcon.entity.v0.feed.ExtractMethod; import org.apache.falcon.entity.v0.process.Input; import org.apache.falcon.entity.v0.process.Output; import org.apache.falcon.entity.v0.process.Process; @@ -169,6 +169,38 @@ private void validateLifecycle(Feed feed) throws FalconException { } } + private void validateLifecycleTags(Feed feed) throws FalconException { + LifecyclePolicyMap map = LifecyclePolicyMap.get(); + for (Cluster cluster : feed.getClusters().getClusters()) { + if (FeedHelper.isLifecycleEnabled(feed, cluster.getName())) { + if ((FeedHelper.getRetentionStage(feed, cluster.getName()) != null) || (FeedHelper.getArchivalStage(feed, cluster.getName()) != null)) { + validateRetentionStage(feed, cluster); + validateArchivalStage(feed, cluster); + for (String policyName : FeedHelper.getPolicies(feed, cluster.getName())) { + map.get(policyName).validate(feed, cluster.getName()); + } + } else { + throw new ValidationException("Atleast one of Retention/Archival is a mandatory stage, didn't find it for cluster: " + + cluster.getName()); + } + } + } + } + + private void validateRetentionStage(Feed feed, Cluster cluster) throws FalconException { + if (FeedHelper.getRetentionStage(feed, cluster.getName()) != null) { + validateRetentionFrequency(feed, cluster.getName()); + } + } + + private void validateArchivalStage(Feed feed, Cluster cluster) throws FalconException { + if (FeedHelper.getArchivalStage(feed, cluster.getName()) != null) { + if (FeedHelper.getArchivalStage(feed, cluster.getName()).getLocation().getPath().isEmpty()) { + throw new ValidationException("Location path cannot be empty."); + } + } + } + private void validateRetentionFrequency(Feed feed, String clusterName) throws FalconException { Frequency retentionFrequency = FeedHelper.getLifecycleRetentionFrequency(feed, clusterName); Frequency feedFrequency = feed.getFrequency(); diff --git a/common/src/main/java/org/apache/falcon/lifecycle/retention/AgeBasedDelete.java b/common/src/main/java/org/apache/falcon/lifecycle/retention/AgeBasedDelete.java index 8d735f97a..30f2f613a 100644 --- a/common/src/main/java/org/apache/falcon/lifecycle/retention/AgeBasedDelete.java +++ b/common/src/main/java/org/apache/falcon/lifecycle/retention/AgeBasedDelete.java @@ -46,7 +46,7 @@ public void validate(Feed feed, String clusterName) throws FalconException { // validate that it is a valid cluster Cluster cluster = FeedHelper.getCluster(feed, clusterName); Frequency retentionLimit = getRetentionLimit(feed, clusterName); - if (cluster != null) { + if (cluster != null && retentionLimit != null) { validateLimitWithSla(feed, cluster, retentionLimit.toString()); validateLimitWithLateData(feed, cluster, retentionLimit.toString()); String lifecycleEngine = StartupProperties.get().getProperty("lifecycle.engine.impl", @@ -122,9 +122,8 @@ public Frequency getRetentionLimit(Feed feed, String clusterName) throws FalconE throw new FalconException("Invalid value for property: " + LIMIT_PROPERTY_NAME + ", should be a valid " + "frequency e.g. hours(2)", e); } - } else { - throw new FalconException("Cluster " + clusterName + " doesn't contain retention stage"); } + return null; } } From ab8eb9029a43bcadb97e8cd54c67407b3124ce1d Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Tue, 21 Nov 2017 17:26:43 +0530 Subject: [PATCH 20/29] Adding policy to startup.properties --- prism/src/test/resources/startup.properties | 8 ++++++-- src/conf/startup.properties | 7 +++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/prism/src/test/resources/startup.properties b/prism/src/test/resources/startup.properties index 63fcd3b70..d66d50534 100644 --- a/prism/src/test/resources/startup.properties +++ b/prism/src/test/resources/startup.properties @@ -59,9 +59,13 @@ # List of Lifecycle policies configured. -*.falcon.feed.lifecycle.policies=org.apache.falcon.lifecycle.retention.AgeBasedDelete +*.falcon.feed.lifecycle.policies=org.apache.falcon.lifecycle.retention.AgeBasedDelete,\ + org.apache.falcon.lifecycle.archival.AgeBasedArchival + # List of builders for the policies. -*.falcon.feed.lifecycle.policy.builders=org.apache.falcon.lifecycle.engine.oozie.retention.AgeBasedDeleteBuilder +*.falcon.feed.lifecycle.policy.builders=org.apache.falcon.lifecycle.engine.oozie.retention.AgeBasedDeleteBuilder,\ + org.apache.falcon.lifecycle.engine.oozie.archival.AgeBasedArchivalBuilder + ##### Falcon Configuration Store Change listeners ##### *.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ org.apache.falcon.entity.ColoClusterRelation,\ diff --git a/src/conf/startup.properties b/src/conf/startup.properties index fae937ac0..8bd2bbe31 100644 --- a/src/conf/startup.properties +++ b/src/conf/startup.properties @@ -104,9 +104,12 @@ prism.application.services=org.apache.falcon.service.LifecyclePolicyMap,\ # List of Lifecycle policies configured. -*.falcon.feed.lifecycle.policies=org.apache.falcon.lifecycle.retention.AgeBasedDelete +*.falcon.feed.lifecycle.policies=org.apache.falcon.lifecycle.retention.AgeBasedDelete,\ + org.apache.falcon.lifecycle.archival.AgeBasedArchival + # List of builders for the policies. -*.falcon.feed.lifecycle.policy.builders=org.apache.falcon.lifecycle.engine.oozie.retention.AgeBasedDeleteBuilder +*.falcon.feed.lifecycle.policy.builders=org.apache.falcon.lifecycle.engine.oozie.retention.AgeBasedDeleteBuilder,\ + org.apache.falcon.lifecycle.engine.oozie.archival.AgeBasedArchivalBuilder ##### Falcon Configuration Store Change listeners ##### *.configstore.listeners=org.apache.falcon.entity.v0.EntityGraph,\ From 73c354ca2fef28cc985ee035c61fe1517ade2360 Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Tue, 21 Nov 2017 18:17:46 +0530 Subject: [PATCH 21/29] Changing validate lifecycle in FeedEntityParser --- .../falcon/entity/parser/FeedEntityParser.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java index 93eba76de..381495c42 100644 --- a/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java +++ b/common/src/main/java/org/apache/falcon/entity/parser/FeedEntityParser.java @@ -154,22 +154,6 @@ public void validate(Feed feed) throws FalconException { } private void validateLifecycle(Feed feed) throws FalconException { - LifecyclePolicyMap map = LifecyclePolicyMap.get(); - for (Cluster cluster : feed.getClusters().getClusters()) { - if (FeedHelper.isLifecycleEnabled(feed, cluster.getName())) { - if (FeedHelper.getRetentionStage(feed, cluster.getName()) == null) { - throw new ValidationException("Retention is a mandatory stage, didn't find it for cluster: " - + cluster.getName()); - } - validateRetentionFrequency(feed, cluster.getName()); - for (String policyName : FeedHelper.getPolicies(feed, cluster.getName())) { - map.get(policyName).validate(feed, cluster.getName()); - } - } - } - } - - private void validateLifecycleTags(Feed feed) throws FalconException { LifecyclePolicyMap map = LifecyclePolicyMap.get(); for (Cluster cluster : feed.getClusters().getClusters()) { if (FeedHelper.isLifecycleEnabled(feed, cluster.getName())) { From e332108e7ffcbff444b94be8ad86baea017f93cb Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Fri, 24 Nov 2017 17:42:59 +0530 Subject: [PATCH 22/29] Adding twiki changes and fixing FeedHelperTest --- common-types/src/main/resources/feed-0.1.xsd | 2 +- .../org/apache/falcon/entity/FeedHelperTest.java | 3 ++- docs/src/site/twiki/EntitySpecification.twiki | 16 +++++++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/common-types/src/main/resources/feed-0.1.xsd b/common-types/src/main/resources/feed-0.1.xsd index 42b07e7aa..2be7cf22d 100644 --- a/common-types/src/main/resources/feed-0.1.xsd +++ b/common-types/src/main/resources/feed-0.1.xsd @@ -588,7 +588,7 @@ - + diff --git a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java index 450b25198..a52103252 100644 --- a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java +++ b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java @@ -266,8 +266,9 @@ public void testGetPolicies() throws Exception { .getParser(EntityType.FEED); Feed feed = parser.parse(this.getClass().getResourceAsStream(FEED3_XML)); List policies = FeedHelper.getPolicies(feed, "testCluster"); - Assert.assertEquals(policies.size(), 1); + Assert.assertEquals(policies.size(), 2); Assert.assertEquals(policies.get(0), "AgeBasedDelete"); + Assert.assertEquals(policies.get(1), "AgeBasedArchival"); } @Test diff --git a/docs/src/site/twiki/EntitySpecification.twiki b/docs/src/site/twiki/EntitySpecification.twiki index 11d1e1b0b..332df28d7 100644 --- a/docs/src/site/twiki/EntitySpecification.twiki +++ b/docs/src/site/twiki/EntitySpecification.twiki @@ -439,13 +439,16 @@ replication. "preserveReplicationNumber" represents preserving replication numbe + + false + default + + -lifecycle tag is the new way to define various stages of a feed's lifecycle. In the example above we have defined a -retention-stage using lifecycle tag. You may define lifecycle at global level or a cluster level or both. Cluster level -configuration takes precedence and falcon falls back to global definition if cluster level specification is missing. +lifecycle tag is the new way to define various stages of a feed's lifecycle. In the example above we have defined retention-stage and archival-stage using lifecycle tag. Atleast one of the satges should be defined if lifecycle tag is used. You may define lifecycle at global level or a cluster level or both. Cluster level configuration takes precedence and falcon falls back to global definition if cluster level specification is missing. ----++++ Retention Stage @@ -468,6 +471,13 @@ wastage of compute resources. In future, we will allow more customisation like customising how to choose instances to be deleted through this method. +----++++ Archival Stage +Lifecycle archival stage allows users to archive data instead of creating dummy replication feed for archival. This is +done through tag in tag. + +In this new method of defining archival you can specify the path at which data is to be archived and the queue for +archival jobs. The behavior of archival-stage is to archive all the data corresponding to the given instance-time. +Tag "location" is a mandatory property and must contain a valid path e.g. "s4://prod/falcon/${YEAR}-${MONTH}-${DAY}" ---++ Process Specification From e107a94b68be927a769defb8924b3661d482a6b6 Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Mon, 27 Nov 2017 21:16:03 +0530 Subject: [PATCH 23/29] Adding test cases for feed archival --- .../org/apache/falcon/entity/FeedHelper.java | 27 ++- .../apache/falcon/entity/FeedHelperTest.java | 198 ++++++++++++++++-- .../entity/parser/FeedEntityParserTest.java | 19 +- .../test/resources/config/feed/feed-0.3.xml | 7 +- .../AgeBasedArchivalCoordinatorBuilder.java | 2 - 5 files changed, 220 insertions(+), 33 deletions(-) diff --git a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java index 2e9a212fc..44cc3a4c6 100644 --- a/common/src/main/java/org/apache/falcon/entity/FeedHelper.java +++ b/common/src/main/java/org/apache/falcon/entity/FeedHelper.java @@ -400,7 +400,7 @@ public static ArchivalStage getArchivalStage(Feed feed, String clusterName) thro if (clusterLifecycle != null && clusterLifecycle.getArchivalStage() != null) { return clusterLifecycle.getArchivalStage(); - } else if (globalLifecycle != null) { + } else if (globalLifecycle != null && globalLifecycle.getArchivalStage() != null) { return globalLifecycle.getArchivalStage(); } } @@ -436,15 +436,20 @@ public static List getPolicies(Feed feed, String clusterName) throws Fal Cluster cluster = getCluster(feed, clusterName); if (cluster != null) { if (isLifecycleEnabled(feed, clusterName)) { - String policy = getRetentionStage(feed, clusterName).getPolicy(); - policy = StringUtils.isBlank(policy) - ? FeedLifecycleStage.RETENTION.getDefaultPolicyName() : policy; - result.add(policy); - - policy = getArchivalStage(feed, clusterName).getPolicy(); - policy = StringUtils.isBlank(policy) - ? FeedLifecycleStage.ARCHIVAL.getDefaultPolicyName() : policy; - result.add(policy); + String policy = ""; + if (getRetentionStage(feed, clusterName) != null) { + policy = getRetentionStage(feed, clusterName).getPolicy(); + policy = StringUtils.isBlank(policy) + ? FeedLifecycleStage.RETENTION.getDefaultPolicyName() : policy; + result.add(policy); + } + + if (getArchivalStage(feed, clusterName) != null) { + policy = getArchivalStage(feed, clusterName).getPolicy(); + policy = StringUtils.isBlank(policy) + ? FeedLifecycleStage.ARCHIVAL.getDefaultPolicyName() : policy; + result.add(policy); + } } return result; } @@ -1306,6 +1311,8 @@ public static String getArchivalPath(Feed feed, String clusterName) throws Falco Location location = archivalStage.getLocation(); if ((location != null) && (location.getPath() != null)) { archivalPath = location.getPath(); + } else { + throw new FalconException("Location cannot be empty."); } } return archivalPath; diff --git a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java index a52103252..fd1e6d458 100644 --- a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java +++ b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java @@ -28,24 +28,7 @@ import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.cluster.Properties; import org.apache.falcon.entity.v0.cluster.Property; -import org.apache.falcon.entity.v0.feed.Argument; -import org.apache.falcon.entity.v0.feed.Arguments; -import org.apache.falcon.entity.v0.feed.ClusterType; -import org.apache.falcon.entity.v0.feed.Clusters; -import org.apache.falcon.entity.v0.feed.Extract; -import org.apache.falcon.entity.v0.feed.ExtractMethod; -import org.apache.falcon.entity.v0.feed.Feed; -import org.apache.falcon.entity.v0.feed.FieldIncludeExclude; -import org.apache.falcon.entity.v0.feed.FieldsType; -import org.apache.falcon.entity.v0.feed.Import; -import org.apache.falcon.entity.v0.feed.Lifecycle; -import org.apache.falcon.entity.v0.feed.Location; -import org.apache.falcon.entity.v0.feed.LocationType; -import org.apache.falcon.entity.v0.feed.Locations; -import org.apache.falcon.entity.v0.feed.MergeType; -import org.apache.falcon.entity.v0.feed.RetentionStage; -import org.apache.falcon.entity.v0.feed.Datasource; -import org.apache.falcon.entity.v0.feed.Validity; +import org.apache.falcon.entity.v0.feed.*; import org.apache.falcon.entity.v0.process.Input; import org.apache.falcon.entity.v0.process.Inputs; import org.apache.falcon.entity.v0.process.Output; @@ -269,6 +252,11 @@ public void testGetPolicies() throws Exception { Assert.assertEquals(policies.size(), 2); Assert.assertEquals(policies.get(0), "AgeBasedDelete"); Assert.assertEquals(policies.get(1), "AgeBasedArchival"); + + feed.getLifecycle().setRetentionStage(null); + policies = FeedHelper.getPolicies(feed, "backupCluster"); + Assert.assertEquals(policies.size(), 1); + Assert.assertEquals(policies.get(0), "AgeBasedArchival"); } @Test @@ -904,6 +892,180 @@ public void testGetRetentionFrequency() throws Exception { new Frequency("hours(4)")); } + @Test + public void testGetArchivalStage() throws Exception { + Feed feed = new Feed(); + feed.setFrequency(new Frequency("hours(1)")); + + // archival stage location is not defined + Lifecycle globalLifecycle = new Lifecycle(); + ArchivalStage globalArchivalStage = new ArchivalStage(); + globalLifecycle.setArchivalStage(globalArchivalStage); + feed.setLifecycle(globalLifecycle); + + Clusters clusters = new Clusters(); + org.apache.falcon.entity.v0.feed.Cluster cluster = new org.apache.falcon.entity.v0.feed.Cluster(); + cluster.setName("cluster1"); + clusters.getClusters().add(cluster); + feed.setClusters(clusters); + + // lifecycle is defined only at global level + Location globalLocation = new Location(); + globalLocation.setType(LocationType.DATA); + globalLocation.setPath("s4://globalPath/archival/"); + globalArchivalStage.setLocation(globalLocation); + globalLifecycle.setArchivalStage(globalArchivalStage); + feed.setLifecycle(globalLifecycle); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + feed.getLifecycle().getArchivalStage().getLocation().getPath()); + + // lifecycle is defined at both global and cluster level + Lifecycle clusterLifecycle = new Lifecycle(); + ArchivalStage clusterArchivalStage = new ArchivalStage(); + Location clusterLocation = new Location(); + clusterLocation.setType(LocationType.DATA); + clusterLocation.setPath("s4://clusterPath/archival/"); + clusterArchivalStage.setLocation(clusterLocation); + clusterLifecycle.setArchivalStage(clusterArchivalStage); + feed.getClusters().getClusters().get(0).setLifecycle(clusterLifecycle); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + cluster.getLifecycle().getArchivalStage().getLocation().getPath()); + + // lifecycle at both level - archival only at cluster level. + feed.getLifecycle().setArchivalStage(null); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + cluster.getLifecycle().getArchivalStage().getLocation().getPath()); + + // lifecycle at both level - archival only at global level. + feed.getLifecycle().setArchivalStage(globalArchivalStage); + feed.getClusters().getClusters().get(0).getLifecycle().setArchivalStage(null); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + feed.getLifecycle().getArchivalStage().getLocation().getPath()); + + // lifecycle is defined only at cluster level + feed.setLifecycle(null); + feed.getClusters().getClusters().get(0).getLifecycle().setArchivalStage(clusterArchivalStage); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + cluster.getLifecycle().getArchivalStage().getLocation().getPath()); + } + + @Test + public void testLifecycleStage() throws Exception { + Feed feed = new Feed(); + feed.setFrequency(new Frequency("days(1)")); + + Clusters clusters = new Clusters(); + org.apache.falcon.entity.v0.feed.Cluster cluster = new org.apache.falcon.entity.v0.feed.Cluster(); + cluster.setName("cluster1"); + clusters.getClusters().add(cluster); + feed.setClusters(clusters); + + // retention stage frequency is not defined + // archival stage location is not defined + Lifecycle globalLifecycle = new Lifecycle(); + + ArchivalStage globalArchivalStage = new ArchivalStage(); + globalLifecycle.setArchivalStage(globalArchivalStage); + RetentionStage globalRetentionStage = new RetentionStage(); + globalLifecycle.setRetentionStage(globalRetentionStage); + feed.setLifecycle(globalLifecycle); + + // lifecycle is defined only at global level + Location globalLocation = new Location(); + globalLocation.setType(LocationType.DATA); + globalLocation.setPath("s4://globalPath/archival/"); + globalArchivalStage.setLocation(globalLocation); + globalLifecycle.setArchivalStage(globalArchivalStage); + globalRetentionStage.setFrequency(new Frequency("hours(2)")); + globalLifecycle.setRetentionStage(globalRetentionStage); + feed.setLifecycle(globalLifecycle); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertNotNull(FeedHelper.getRetentionStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + feed.getLifecycle().getArchivalStage().getLocation().getPath()); + Assert.assertEquals(FeedHelper.getLifecycleRetentionFrequency(feed, cluster.getName()), + feed.getLifecycle().getRetentionStage().getFrequency()); + + // lifecycle is defined at both global and cluster level + Lifecycle clusterLifecycle = new Lifecycle(); + ArchivalStage clusterArchivalStage = new ArchivalStage(); + Location clusterLocation = new Location(); + clusterLocation.setType(LocationType.DATA); + clusterLocation.setPath("s4://clusterPath/archival/"); + clusterArchivalStage.setLocation(clusterLocation); + clusterLifecycle.setArchivalStage(clusterArchivalStage); + RetentionStage clusterRetentionStage = new RetentionStage(); + clusterRetentionStage.setFrequency(new Frequency("hours(4)")); + clusterLifecycle.setRetentionStage(clusterRetentionStage); + feed.getClusters().getClusters().get(0).setLifecycle(clusterLifecycle); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + cluster.getLifecycle().getArchivalStage().getLocation().getPath()); + Assert.assertNotNull(FeedHelper.getRetentionStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getLifecycleRetentionFrequency(feed, cluster.getName()), + cluster.getLifecycle().getRetentionStage().getFrequency()); + + // lifecycle at both level - retention/archival only at cluster level. + feed.getLifecycle().setArchivalStage(null); + feed.getLifecycle().setRetentionStage(null); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + cluster.getLifecycle().getArchivalStage().getLocation().getPath()); + Assert.assertNotNull(FeedHelper.getRetentionStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getLifecycleRetentionFrequency(feed, cluster.getName()), + cluster.getLifecycle().getRetentionStage().getFrequency()); + + // lifecycle at both level - retention/archival only at global level. + feed.getLifecycle().setArchivalStage(globalArchivalStage); + feed.getLifecycle().setRetentionStage(globalRetentionStage); + feed.getClusters().getClusters().get(0).getLifecycle().setArchivalStage(null); + feed.getClusters().getClusters().get(0).getLifecycle().setRetentionStage(null); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + feed.getLifecycle().getArchivalStage().getLocation().getPath()); + Assert.assertNotNull(FeedHelper.getRetentionStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getLifecycleRetentionFrequency(feed, cluster.getName()), + feed.getLifecycle().getRetentionStage().getFrequency()); + + // lifecycle at both level- retention at cluster level, archival at global level + feed.getClusters().getClusters().get(0).getLifecycle().setRetentionStage(clusterRetentionStage); + feed.getLifecycle().setRetentionStage(null); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + feed.getLifecycle().getArchivalStage().getLocation().getPath()); + Assert.assertNotNull(FeedHelper.getRetentionStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getLifecycleRetentionFrequency(feed, cluster.getName()), + cluster.getLifecycle().getRetentionStage().getFrequency()); + + // lifecycle at both level- retention at global level, archival at cluster level + feed.getClusters().getClusters().get(0).getLifecycle().setArchivalStage(clusterArchivalStage); + feed.getClusters().getClusters().get(0).getLifecycle().setRetentionStage(null); + feed.getLifecycle().setRetentionStage(globalRetentionStage); + feed.getLifecycle().setArchivalStage(null); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + cluster.getLifecycle().getArchivalStage().getLocation().getPath()); + Assert.assertNotNull(FeedHelper.getRetentionStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getLifecycleRetentionFrequency(feed, cluster.getName()), + feed.getLifecycle().getRetentionStage().getFrequency()); + + // lifecycle is defined only at cluster level + feed.setLifecycle(null); + feed.getClusters().getClusters().get(0).getLifecycle().setArchivalStage(clusterArchivalStage); + feed.getClusters().getClusters().get(0).getLifecycle().setRetentionStage(clusterRetentionStage); + Assert.assertNotNull(FeedHelper.getArchivalStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getArchivalPath(feed, cluster.getName()), + cluster.getLifecycle().getArchivalStage().getLocation().getPath()); + Assert.assertNotNull(FeedHelper.getRetentionStage(feed, cluster.getName())); + Assert.assertEquals(FeedHelper.getLifecycleRetentionFrequency(feed, cluster.getName()), + cluster.getLifecycle().getRetentionStage().getFrequency()); + } + @Test public void testFeedImportSnapshot() throws Exception { Cluster cluster = publishCluster(); diff --git a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java index ced4fc502..559b7116c 100644 --- a/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java +++ b/common/src/test/java/org/apache/falcon/entity/parser/FeedEntityParserTest.java @@ -187,14 +187,20 @@ public void testLifecycleParse() throws Exception { assertEquals("AgeBasedDelete", FeedHelper.getPolicies(feed, "testCluster").get(0)); assertEquals("reports", feed.getLifecycle().getRetentionStage().getQueue()); assertEquals("NORMAL", feed.getLifecycle().getRetentionStage().getPriority()); + assertEquals("default", feed.getLifecycle().getArchivalStage().getQueue()); + assertEquals("s4://prod/falcon/${YEAR}-${MONTH}-${DAY}-${HOUR}/", feed.getLifecycle().getArchivalStage().getLocation().getPath()); + assertEquals("AgeBasedArchival", FeedHelper.getPolicies(feed, "testCluster").get(1)); + assertEquals("s4://prodnew/falcon/${YEAR}-${MONTH}-${DAY}-${HOUR}/", feed.getClusters().getClusters().get(0).getLifecycle().getArchivalStage().getLocation().getPath()); } @Test(expectedExceptions = ValidationException.class, - expectedExceptionsMessageRegExp = ".*Retention is a mandatory stage.*") - public void testMandatoryRetention() throws Exception { + expectedExceptionsMessageRegExp = ".*Atleast one of Retention/Archival is a mandatory stage, didn't find it for cluster.*") + public void testLifecycleStage() throws Exception { Feed feed = parser.parseAndValidate(this.getClass() .getResourceAsStream(FEED3_XML)); feed.getLifecycle().setRetentionStage(null); + feed.getLifecycle().setArchivalStage(null); + assertEquals("AgeBasedDelete", FeedHelper.getPolicies(feed, "testCluster").get(0)); parser.validate(feed); } @@ -253,6 +259,15 @@ public void testRetentionFrequency() throws Exception { parser.validate(feed); } + @Test(expectedExceptions = ValidationException.class, + expectedExceptionsMessageRegExp = ".*Location path cannot be empty.*") + public void testArchivalPath() throws Exception { + Feed feed = parser.parseAndValidate(this.getClass() + .getResourceAsStream(FEED3_XML)); + feed.getLifecycle().getArchivalStage().getLocation().setPath(""); + parser.validate(feed); + } + @Test(expectedExceptions = ValidationException.class) public void applyValidationInvalidFeed() throws Exception { Feed feed = parser.parseAndValidate(ProcessEntityParserTest.class diff --git a/common/src/test/resources/config/feed/feed-0.3.xml b/common/src/test/resources/config/feed/feed-0.3.xml index 360b1d436..ccfd0dad1 100644 --- a/common/src/test/resources/config/feed/feed-0.3.xml +++ b/common/src/test/resources/config/feed/feed-0.3.xml @@ -53,6 +53,11 @@ + + false + default + + @@ -82,7 +87,7 @@ false default - + diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java index d27a2dc23..f35245323 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java @@ -37,9 +37,7 @@ import org.apache.falcon.expression.ExpressionHelper; import org.apache.falcon.lifecycle.engine.oozie.utils.OozieBuilderUtils; import org.apache.falcon.oozie.coordinator.*; -import org.apache.falcon.util.RuntimeProperties; import org.apache.falcon.workflow.WorkflowExecutionArgs; -import org.apache.falcon.workflow.WorkflowExecutionContext; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 38a70c3ae510c78a4bfe71978b715498b07fa8d2 Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Thu, 30 Nov 2017 15:45:42 +0530 Subject: [PATCH 24/29] Setting datasets in AgeBasedArchivalCoordinatorBuilder --- .../AgeBasedArchivalCoordinatorBuilder.java | 81 ++++++++++++++++--- 1 file changed, 70 insertions(+), 11 deletions(-) diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java index f35245323..7ffe5baf7 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalCoordinatorBuilder.java @@ -19,11 +19,9 @@ package org.apache.falcon.lifecycle.engine.oozie.archival; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.DateUtils; import org.apache.falcon.FalconException; import org.apache.falcon.LifeCycle; import org.apache.falcon.Tag; -import org.apache.falcon.entity.CatalogStorage; import org.apache.falcon.entity.EntityUtil; import org.apache.falcon.entity.FeedHelper; import org.apache.falcon.entity.Storage; @@ -32,11 +30,19 @@ import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.entity.v0.feed.LocationType; -import org.apache.falcon.entity.v0.feed.Property; import org.apache.falcon.entity.v0.process.ExecutionType; import org.apache.falcon.expression.ExpressionHelper; import org.apache.falcon.lifecycle.engine.oozie.utils.OozieBuilderUtils; -import org.apache.falcon.oozie.coordinator.*; +import org.apache.falcon.oozie.coordinator.SYNCDATASET; +import org.apache.falcon.oozie.coordinator.DATAOUT; +import org.apache.falcon.oozie.coordinator.DATAIN; +import org.apache.falcon.oozie.coordinator.COORDINATORAPP; +import org.apache.falcon.oozie.coordinator.ACTION; +import org.apache.falcon.oozie.coordinator.WORKFLOW; +import org.apache.falcon.oozie.coordinator.DATASETS; +import org.apache.falcon.oozie.coordinator.CONTROLS; +import org.apache.falcon.oozie.coordinator.INPUTEVENTS; +import org.apache.falcon.oozie.coordinator.OUTPUTEVENTS; import org.apache.falcon.workflow.WorkflowExecutionArgs; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; @@ -44,8 +50,6 @@ import java.util.*; -import static org.json.XMLTokener.entity; - public class AgeBasedArchivalCoordinatorBuilder { private AgeBasedArchivalCoordinatorBuilder() { @@ -57,6 +61,10 @@ private AgeBasedArchivalCoordinatorBuilder() { private static final String PARALLEL = "parallel"; private static final String TIMEOUT = "timeout"; private static final String ORDER = "order"; + public static final String IN_DATASET_NAME = "input-dataset"; + public static final String OUT_DATASET_NAME = "output-dataset"; + public static final String DATAIN_NAME = "input"; + public static final String DATAOUT_NAME = "output"; /** * Builds the coordinator app. @@ -77,7 +85,7 @@ public static Properties build(Cluster cluster, Path basePath, Feed feed, Proper COORDINATORAPP coord = new COORDINATORAPP(); - String coordName = EntityUtil.getWorkflowName(LifeCycle.EVICTION.getTag(), feed).toString(); + String coordName = EntityUtil.getWorkflowName(LifeCycle.ARCHIVAL.getTag(), feed).toString(); long replicationDelayInMillis = getReplicationDelayInMillis(feed, cluster); Date sourceStartDate = getStartDate(feed, cluster, replicationDelayInMillis); @@ -93,8 +101,7 @@ public static Properties build(Cluster cluster, Path basePath, Feed feed, Proper long delayInMins = -1 * replicationDelayInMillis / (1000 * 60); String elExp = "${now(0," + delayInMins + ")}"; - coord.getInputEvents().getDataIn().get(0).getInstance().set(0, elExp); - coord.getOutputEvents().getDataOut().get(0).setInstance(elExp); + initializeInputOutputPath(coord, cluster, feed, elExp); } setCoordControls(feed, coord); @@ -115,6 +122,45 @@ public static Properties build(Cluster cluster, Path basePath, Feed feed, Proper } + private static void initializeInputOutputPath(COORDINATORAPP coord, Cluster cluster, Feed feed, String elExp) + throws FalconException { + + if (coord.getInputEvents() == null) { + coord.setInputEvents(new INPUTEVENTS()); + } + + if (coord.getOutputEvents() == null) { + coord.setOutputEvents(new OUTPUTEVENTS()); + } + + DATAIN datain = createDataIn(feed, cluster); + coord.getInputEvents().getDataIn().add(datain); + + DATAOUT dataout = createDataOut(feed, cluster); + coord.getOutputEvents().getDataOut().add(dataout); + + coord.getInputEvents().getDataIn().get(0).getInstance().set(0, elExp); + coord.getOutputEvents().getDataOut().get(0).setInstance(elExp); + } + + + private static DATAIN createDataIn(Feed feed, Cluster cluster) { + DATAIN datain = new DATAIN(); + datain.setName(DATAIN_NAME); + datain.setDataset(IN_DATASET_NAME); + org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName()); + datain.getInstance().add(SchemaHelper.formatDateUTC(feedCluster.getValidity().getStart())); + return datain; + } + + private static DATAOUT createDataOut(Feed feed, Cluster cluster) { + DATAOUT dataout = new DATAOUT(); + dataout.setName(DATAOUT_NAME); + dataout.setDataset(OUT_DATASET_NAME); + org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName()); + dataout.setInstance("${coord:current(0)}"); + return dataout; + } private static ACTION getReplicationWorkflowAction(Feed feed, Cluster cluster, Path buildPath, String coordName, Storage sourceStorage) throws FalconException { @@ -235,7 +281,12 @@ private static void setCoordControls(Feed feed, COORDINATORAPP coord) throws Fal } private static void initializeInputDataSet(Feed feed, Cluster cluster, COORDINATORAPP coord, Storage storage) throws FalconException { - SYNCDATASET inputDataset = (SYNCDATASET)coord.getDatasets().getDatasetOrAsyncDataset().get(0); + if (coord.getDatasets() == null) { + coord.setDatasets(new DATASETS()); + } + + SYNCDATASET inputDataset = new SYNCDATASET(); + inputDataset.setName(IN_DATASET_NAME); String uriTemplate = storage.getUriTemplate(LocationType.DATA); inputDataset.setUriTemplate(uriTemplate); @@ -247,14 +298,22 @@ private static void initializeInputDataSet(Feed feed, Cluster cluster, COORDINAT } else { inputDataset.setDoneFlag(feed.getAvailabilityFlag()); } + + coord.getDatasets().getDatasetOrAsyncDataset().add(inputDataset); } private static void initializeOutputDataSet(Feed feed, Cluster cluster, COORDINATORAPP coord, String targetPath) throws FalconException { - SYNCDATASET outputDataset = (SYNCDATASET)coord.getDatasets().getDatasetOrAsyncDataset().get(1); + if (coord.getDatasets() == null) { + coord.setDatasets(new DATASETS()); + } + + SYNCDATASET outputDataset = new SYNCDATASET(); + outputDataset.setName(OUT_DATASET_NAME); outputDataset.setUriTemplate(targetPath); setDatasetValues(feed, outputDataset, cluster); + coord.getDatasets().getDatasetOrAsyncDataset().add(outputDataset); } From 0bbe6e48f1770e5e90826eabf0fdd47dfe8a5a1b Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Thu, 30 Nov 2017 16:11:38 +0530 Subject: [PATCH 25/29] Minor nit --- .../java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java index f6b18af39..e094e0970 100644 --- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java +++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedBundleBuilder.java @@ -58,7 +58,8 @@ protected List buildCoords(Cluster cluster, Path buildPath) throws F props.add(appProps); } } - } else if (FeedHelper.getRetentionStage(this.entity, cluster.getName()) == null){ + } + if (FeedHelper.getRetentionStage(this.entity, cluster.getName()) == null){ List evictionProps = OozieCoordinatorBuilder.get(entity, Tag.RETENTION).buildCoords(cluster, buildPath); if (evictionProps != null) { From 724f8b541f8f7e6c867021c9dd5c963d0701442a Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Mon, 4 Dec 2017 12:36:58 +0530 Subject: [PATCH 26/29] Adding EntityOperations.ARCHIVE --- .../org/apache/falcon/workflow/WorkflowExecutionContext.java | 2 +- .../engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/falcon/workflow/WorkflowExecutionContext.java b/common/src/main/java/org/apache/falcon/workflow/WorkflowExecutionContext.java index d8040f08d..30731ec3d 100644 --- a/common/src/main/java/org/apache/falcon/workflow/WorkflowExecutionContext.java +++ b/common/src/main/java/org/apache/falcon/workflow/WorkflowExecutionContext.java @@ -75,7 +75,7 @@ public enum Type {PRE_PROCESSING, POST_PROCESSING, WORKFLOW_JOB, COORDINATOR_ACT * Entity operations supported. */ public enum EntityOperations { - GENERATE, DELETE, REPLICATE, IMPORT, EXPORT + GENERATE, DELETE, REPLICATE, IMPORT, EXPORT, ARCHIVE } public static final WorkflowExecutionArgs[] USER_MESSAGE_ARGS = { diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java index ee45666a7..59312a7c7 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java @@ -82,7 +82,7 @@ public static Properties build(Cluster cluster, Path basePath, Feed feed) throws Properties props = OozieBuilderUtils.getProperties(buildPath, wfName); - props.putAll(OozieBuilderUtils.createDefaultConfiguration(cluster, feed, WorkflowExecutionContext.EntityOperations.REPLICATE)); + props.putAll(OozieBuilderUtils.createDefaultConfiguration(cluster, feed, WorkflowExecutionContext.EntityOperations.ARCHIVE)); props.putAll(getWorkflowProperties(feed, cluster)); props.putAll(FeedHelper.getUserWorkflowProperties(LifeCycle.ARCHIVAL)); From 16955ca76b040f6038d9cd63a0b3eb7c14d99ee4 Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Mon, 4 Dec 2017 15:13:42 +0530 Subject: [PATCH 27/29] Fixing path in workflow builder --- .../oozie/archival/AgeBasedArchivalWorkflowBuilder.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java index 59312a7c7..67058f93d 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java @@ -54,7 +54,11 @@ private AgeBasedArchivalWorkflowBuilder(){ } public static Properties build(Cluster cluster, Path basePath, Feed feed) throws FalconException { - Path buildPath = OozieBuilderUtils.getBuildPath(basePath, LifeCycle.ARCHIVAL.getTag()); + org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(feed, cluster.getName()); + + // workflow is serialized to a specific dir + Path buildPath = new Path(basePath, Tag.ARCHIVAL.name() + "/" + feedCluster.getName()); + WORKFLOWAPP workflow = new WORKFLOWAPP(); String wfName = EntityUtil.getWorkflowName(Tag.ARCHIVAL, feed).toString(); From 098548fe46bbaa0ddb9758a867cf5c5b34843d9f Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Mon, 4 Dec 2017 15:24:56 +0530 Subject: [PATCH 28/29] Minor checkstyle fix --- .../apache/falcon/entity/FeedHelperTest.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java index fd1e6d458..ffbace661 100644 --- a/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java +++ b/common/src/test/java/org/apache/falcon/entity/FeedHelperTest.java @@ -28,7 +28,25 @@ import org.apache.falcon.entity.v0.cluster.Cluster; import org.apache.falcon.entity.v0.cluster.Properties; import org.apache.falcon.entity.v0.cluster.Property; -import org.apache.falcon.entity.v0.feed.*; +import org.apache.falcon.entity.v0.feed.Arguments; +import org.apache.falcon.entity.v0.feed.Argument; +import org.apache.falcon.entity.v0.feed.ClusterType; +import org.apache.falcon.entity.v0.feed.Clusters; +import org.apache.falcon.entity.v0.feed.Extract; +import org.apache.falcon.entity.v0.feed.ExtractMethod; +import org.apache.falcon.entity.v0.feed.Feed; +import org.apache.falcon.entity.v0.feed.FieldIncludeExclude; +import org.apache.falcon.entity.v0.feed.FieldsType; +import org.apache.falcon.entity.v0.feed.Import; +import org.apache.falcon.entity.v0.feed.Locations; +import org.apache.falcon.entity.v0.feed.Location; +import org.apache.falcon.entity.v0.feed.LocationType; +import org.apache.falcon.entity.v0.feed.Lifecycle; +import org.apache.falcon.entity.v0.feed.MergeType; +import org.apache.falcon.entity.v0.feed.RetentionStage; +import org.apache.falcon.entity.v0.feed.ArchivalStage; +import org.apache.falcon.entity.v0.feed.Datasource; +import org.apache.falcon.entity.v0.feed.Validity; import org.apache.falcon.entity.v0.process.Input; import org.apache.falcon.entity.v0.process.Inputs; import org.apache.falcon.entity.v0.process.Output; From 3eb2109ec41d457ed73aeb9cc9686b903344f92d Mon Sep 17 00:00:00 2001 From: Pragya Mittal Date: Mon, 4 Dec 2017 15:28:45 +0530 Subject: [PATCH 29/29] Minor checkstyle fix --- .../engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java index 67058f93d..5ba60c50b 100644 --- a/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java +++ b/lifecycle/src/main/java/org/apache/falcon/lifecycle/engine/oozie/archival/AgeBasedArchivalWorkflowBuilder.java @@ -30,7 +30,8 @@ import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.entity.v0.feed.Property; import org.apache.falcon.lifecycle.engine.oozie.utils.OozieBuilderUtils; -import org.apache.falcon.oozie.workflow.*; +import org.apache.falcon.oozie.workflow.WORKFLOWAPP; +import org.apache.falcon.oozie.workflow.ACTION; import org.apache.falcon.util.ReplicationDistCpOption; import org.apache.falcon.util.RuntimeProperties; import org.apache.falcon.workflow.WorkflowExecutionArgs;