Feature: Add VSTS CI (#561)

* fix job submission bug and integration tests * merge * update tests, add vsts-ci.yml * add python step * debug statements * update build * update build add print * build update * fix bug * debug * undo * parallelize build * add trigger * typo * remove env * whitespace * whitespace * whitespace * remove debug branch
Azure · May 22, 2018 · 66037fd · 66037fd
1 parent 1527929
commit 66037fd
Show file tree

Hide file tree

Showing 8 changed files with 453 additions and 355 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -35,7 +35,7 @@
             "spark", "cluster", "create", "--id", "spark-debug"
           ],
           "env": {},
-          "envFile": "${workspaceFolder}/.env",
+          "envFile": "${workspaceFolder}/.venv",
           "debugOptions": [
               "RedirectOutput"
           ]

diff --git a/.vsts-ci.yml b/.vsts-ci.yml
@@ -0,0 +1,20 @@
+trigger:
+  - master
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '>= 3.5' 
+    addToPath: true 
+    architecture: 'x64'
+
+- script: |
+    pip install -r requirements.txt
+    pip install -e .
+  condition: and(succeeded(), eq(variables['agent.os'], 'linux'))
+  displayName: install aztk
+
+- script: |
+    pytest -n 50
+  condition: and(succeeded(), in(variables['agent.os'], 'linux'))
+  displayName: pytest
diff --git a/aztk/spark/helpers/job_submission.py b/aztk/spark/helpers/job_submission.py
@@ -22,7 +22,8 @@ def __app_cmd():
     docker_exec.add_argument("spark /bin/bash >> output.log 2>&1 -c \"" \
                              "source ~/.bashrc; " \
                              "export PYTHONPATH=$PYTHONPATH:\$AZTK_WORKING_DIR; " \
-                             "$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/job_submission.py\"")
+                             "cd \$AZ_BATCH_TASK_WORKING_DIR; " \
+                             "\$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/job_submission.py\"")
     return docker_exec.to_str()
 
 

diff --git a/aztk/spark/helpers/submit.py b/aztk/spark/helpers/submit.py
@@ -84,7 +84,7 @@ def generate_task(spark_client, container_id, application):
     task_cmd.add_argument('spark /bin/bash >> output.log 2>&1')
     task_cmd.add_argument('-c "source ~/.bashrc; ' \
                           'export PYTHONPATH=$PYTHONPATH:\$AZTK_WORKING_DIR; ' \
-                          'cd $AZ_BATCH_TASK_WORKING_DIR; ' \
+                          'cd \$AZ_BATCH_TASK_WORKING_DIR; ' \
                           '\$AZTK_WORKING_DIR/.aztk-env/.venv/bin/python \$AZTK_WORKING_DIR/aztk/node_scripts/submit.py"')
 
     # Create task

diff --git a/aztk/spark/models/models.py b/aztk/spark/models/models.py
@@ -8,6 +8,7 @@
 class SparkToolkit(aztk.models.Toolkit):
     def __init__(self, version: str, environment: str = None, environment_version: str = None):
         super().__init__(
+            software="spark",
             version=version,
             environment=environment,
             environment_version=environment_version,

diff --git a/aztk/spark/utils/util.py b/aztk/spark/utils/util.py
@@ -45,4 +45,3 @@ def wait_for_master_to_be_ready(client, cluster_id: str):
                     "Master didn't become ready before timeout.")
 
             time.sleep(10)
-    time.sleep(5)