Merge pull request #1026 from hiroyuki-sato/validate-project-on-serve…

…r-side

Digdag push validation on the server side.

digdag-server/src/main/java/io/digdag/server/rs/ProjectResource.java

@@ -74,6 +74,8 @@
 import io.digdag.core.repository.StoredProject;
 import io.digdag.core.repository.StoredRevision;
 import io.digdag.core.repository.StoredWorkflowDefinition;
+import io.digdag.core.repository.WorkflowDefinition;
+import io.digdag.core.repository.WorkflowDefinitionList;
 import io.digdag.core.schedule.ScheduleStore;
 import io.digdag.core.schedule.ScheduleStoreManager;
 import io.digdag.core.schedule.SchedulerManager;
@@ -82,7 +84,9 @@
 import io.digdag.core.session.SessionStoreManager;
 import io.digdag.core.session.StoredSessionWithLastAttempt;
 import io.digdag.core.storage.ArchiveManager;
+import io.digdag.core.workflow.Workflow;
 import io.digdag.core.workflow.WorkflowCompiler;
+import io.digdag.core.workflow.WorkflowTask;
 import io.digdag.server.GenericJsonExceptionHandler;
 import io.digdag.spi.DirectDownloadHandle;
 import io.digdag.spi.SecretControlStore;
@@ -549,6 +553,8 @@ public RestProject putProject(@QueryParam("project") String name, @QueryParam("r
                     if (md5Count.getCount() != contentLength) {
                         throw new IllegalArgumentException("Content-Length header doesn't match with uploaded data size");
                     }
+
+                    validateWorkflowAndSchedule(meta);
                 }
 
                 ArchiveManager.Location location =
@@ -681,6 +687,26 @@ private void validateTarEntry(TarArchiveEntry entry)
         }
     }
 
+    private void validateWorkflowAndSchedule(ArchiveMetadata meta)
+    {
+        WorkflowDefinitionList defs = meta.getWorkflowList();
+        for (WorkflowDefinition def : defs.get()) {
+            Workflow wf = compiler.compile(def.getName(), def.getConfig());
+
+            // validate workflow and schedule
+            for (WorkflowTask task : wf.getTasks()) {
+                // raise an exception if task doesn't valid.
+                task.getConfig();
+            }
+            Revision rev = Revision.builderFromArchive("check", meta, getUserInfo())
+                    .archiveType(ArchiveType.NONE)
+                    .build();
+            // raise an exception if "schedule:" is invalid.
+            srm.tryGetScheduler(rev, def);
+
+        }
+    }
+
     @PUT
     @Consumes("application/json")
     @Path("/api/projects/{id}/secrets/{key}")

digdag-tests/src/test/java/acceptance/ValidateProjectIT.java

@@ -0,0 +1,87 @@
+package acceptance;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import utils.CommandStatus;
+import utils.TemporaryDigdagServer;
+
+import java.nio.file.Path;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.is;
+import static utils.TestUtils.copyResource;
+import static utils.TestUtils.main;
+
+//
+// This file doesn't contain normal case.
+// It defined in another test.
+//
+public class ValidateProjectIT
+{
+    @Rule
+    public TemporaryFolder folder = new TemporaryFolder();
+
+    @Rule
+    public TemporaryDigdagServer server = TemporaryDigdagServer.builder()
+            .build();
+
+    private Path config;
+    private Path projectDir;
+
+    @Before
+    public void setUp()
+            throws Exception
+    {
+        projectDir = folder.getRoot().toPath().resolve("foobar");
+        config = folder.newFile().toPath();
+    }
+
+    @Test
+    public void uploadInvalidTaskProject()
+            throws Exception
+    {
+        // Create new project
+        CommandStatus initStatus = main("init",
+                "-c", config.toString(),
+                projectDir.toString());
+        assertThat(initStatus.code(), is(0));
+
+        copyResource("acceptance/error_task/invalid_at_group.dig", projectDir.resolve("invalid_at_group.dig"));
+
+        // Push the project
+        CommandStatus pushStatus = main(
+                "push",
+                "--project", projectDir.toString(),
+                "foobar",
+                "-c", config.toString(),
+                "-e", server.endpoint());
+        assertThat(pushStatus.code(), is(1));
+        assertThat(pushStatus.errUtf8(), containsString("A task can't have more than one operator"));
+    }
+
+    @Test
+    public void uploadInvalidScheduleProject()
+            throws Exception
+    {
+        // Create new project
+        CommandStatus initStatus = main("init",
+                "-c", config.toString(),
+                projectDir.toString());
+        assertThat(initStatus.code(), is(0));
+
+        copyResource("acceptance/schedule/invalid_schedule.dig", projectDir.resolve("invalid_schedule.dig"));
+
+        // Push the project
+        CommandStatus pushStatus = main(
+                "push",
+                "--project", projectDir.toString(),
+                "foobar",
+                "-c", config.toString(),
+                "-e", server.endpoint());
+        assertThat(pushStatus.code(), is(1));
+        assertThat(pushStatus.errUtf8(), containsString("scheduler requires mm:ss format"));
+    }
+}

digdag-tests/src/test/resources/acceptance/schedule/invalid_schedule.dig

@@ -0,0 +1,8 @@
+timezone: UTC
+
+schedule:
+  hourly>: "10:11:12" # correct format is "MM:SS"
+
++foo:
+  sh>: "touch foo.out"
+