From 81326be365a6e71d3f89fd0ac0e00e99b1ecba81 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Thu, 17 Oct 2019 17:39:58 -0700 Subject: [PATCH] Revert "Support subdirectories for junit files and grouping in test grid by name (#490)" (#493) This reverts commit 16ea186441a45d8e74457317a177d232eea75a71. The commit breaks kfctl_e2e_workflow.py tests because we introduced an extra argument. --- README.md | 114 ------ package-lock.json | 466 ------------------------ py/kubeflow/testing/argo_build_util.py | 1 - py/kubeflow/testing/ci/kf_unittests.py | 60 +-- py/kubeflow/testing/conftest.py | 24 -- py/kubeflow/testing/prow_artifacts.py | 18 +- py/kubeflow/testing/run_e2e_workflow.py | 13 - py/kubeflow/testing/test_py_lint.py | 64 ++-- py/kubeflow/testing/util.py | 36 +- 9 files changed, 61 insertions(+), 735 deletions(-) delete mode 100644 package-lock.json delete mode 100644 py/kubeflow/testing/conftest.py diff --git a/README.md b/README.md index 42ad6ea45..4db855cb0 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,6 @@ - [Argo UI](#argo-ui) - [Stackdriver logs](#stackdriver-logs) - [Debugging Failed Tests](#debugging-failed-tests) - - [Logs and Cluster Access for Kubeflow CI](#logs-and-cluster-access-for-kubeflow-ci) - - [Access Control](#access-control) - [No results show up in Gubernator](#no-results-show-up-in-gubernator) - [No Logs in Argo UI For Step or Pod Id missing in Argo Logs](#no-logs-in-argo-ui-for-step-or-pod-id-missing-in-argo-logs) - [Debugging Failed Deployments](#debugging-failed-deployments) @@ -32,9 +30,6 @@ - [Setting up a Kubeflow Repository to Use Prow ](#setting-up-a-kubeflow-repository-to-use-prow-a-idprow-setupa) - [Writing An Argo Workflow For An E2E Test](#writing-an-argo-workflow-for-an-e2e-test) - [Adding an E2E test to a repository](#adding-an-e2e-test-to-a-repository) - - [Python function](#python-function) - - [ksonnet](#ksonnet) - - [Using pytest to write tests](#using-pytest-to-write-tests) - [Prow Variables](#prow-variables) - [Argo Spec](#argo-spec) - [Creating K8s resources in tests.](#creating-k8s-resources-in-tests) @@ -181,65 +176,6 @@ gcloud one liners for fetching logs. ## Debugging Failed Tests -### Logs and Cluster Access for Kubeflow CI - -Our tests are split across three projects - -* **k8s-prow-builds** - - * This is owned by the prow team - * This is where the prow jobs run - * We are working on changing this see [kubeflow/testing#475](https://github.com/kubeflow/testing/issues/475) - -* **kubeflow-ci** - - * This is where the Argo E2E workflows kicked off by the prow jobs run - * This is where other Kubeflow test infra (e.g. various cron jobs run) - -* **kubeflow-ci-deployment** - - * This is the project where E2E tests actually create Kubeflow clusters - - -#### Access Control - -We currently have the following levels of access - -* **ci-viewer-only** - - * This is controlled by the group [ci-viewer](https://github.com/kubeflow/internal-acls/blob/master/ci-viewer.members.txt) - - * This group basically grants viewer only access to projects **kubeflow-ci** and **kubeflow-ci-deployment** - * This provides access to stackdriver for both projects - - * Folks making regular and continual contributions to Kubeflow and in need of access to debug - tests can generally have access - -* **ci-edit/admin** - - * This is controlled by the group [ci-team](https://github.com/kubeflow/internal-acls/blob/master/ci-team.members.txt) - - * This group grants permissions necessary to administer the infrastructure running in **kubeflow-ci** and **kubeflow-ci-deployment** - - * Access to this group is highly restricted since this is critical infrastructure for the project - - * Following standard operating procedures we want to limit the number of folks with direct access to infrastructure - - * Rather than granting more people access we want to develop scalable practices that eliminate the need for - granting large numbers of people access (e.g. developing git ops processes) - - * **example-maintainers** - - * This is controlled by the group [example-maintainers](https://github.com/kubeflow/internal-acls/blob/master/example-maintainers.members.txt) - - * This group provides more direct access to the Kubeflow clusters running **kubeflow-ci-deployment** - - * This group is intended for the folks actively developing and maintaining tests for Kubeflow examples - - * Continuous testing for kubeflow examples should run against regularly updated, auto-deployed clusters in project **kubeflow-ci-deployment** - - * Example maintainers are granted elevated access to these clusters in order to facilitate development of these tests - ### No results show up in Gubernator If no results show up in Gubernator this means the prow job didn't get far enough to upload any results/logs to GCS. @@ -274,10 +210,6 @@ To access the stackdriver logs ### No Logs in Argo UI For Step or Pod Id missing in Argo Logs -The Argo UI will surface logs for the pod but only if the pod hasn't been deleted yet by Kubernetes. - -Using stackdriver to fetch pod logs is more reliable/durable but requires viewer permissions for Kubeflow's ci's infrastructure. - An Argo workflow fails and you click on the failed step in the Argo UI to get the logs and you see the error @@ -863,52 +795,6 @@ Follow these steps to add a new test to a repository. * **params**: A dictionary of parameters to set on the ksonnet component e.g. by running `ks param set ${COMPONENT} ${PARAM_NAME} ${PARAM_VALUE}` -### Using pytest to write tests - -* [pytest](https://docs.pytest.org/en/latest/) is really useful for writing tests - - * Results can be emitted as junit files which is what prow needs to report test results - * It provides [annotations](http://doc.pytest.org/en/latest/skipping.html) to skip tests or mark flaky tests as expected to fail - -* Use pytest to easily script various checks - - * For example [kf_is_ready_test.py](https://github.com/kubeflow/kubeflow/blob/master/testing/kfctl/kf_is_ready_test.py) - uses some simple scripting to test that various K8s objects are deployed and healthy - -* Pytest provides fixtures for setting additional attributes in the junit files ([docs](http://doc.pytest.org/en/latest/usage.html)) - - * In particular [record_xml_attribute](http://doc.pytest.org/en/latest/usage.html#record-xml-attribute) allows us to set attributes - that control how's the results are grouped in test grid - - * **name** - This is the name shown in test grid - - * Testgrid supports [grouping](https://github.com/kubernetes/test-infra/tree/master/testgrid#grouping-tests) by spliting the tests into a hierarchy based on the name - - * **recommendation** Leverage this feature to name tests to support grouping; e.g. use the pattern - - ``` - {WORKFLOW_NAME}/{PY_FUNC_NAME} - ``` - - * **workflow_name** Workflow name as set in prow_config.yaml - * **PY_FUNC_NAME** the name of the python test function - - * util.py provides the helper method `set_pytest_junit` to set the required attributes - * run_e2e_workflow.py will pass the argument `test_target_name` to your py function to create the Argo workflow - - * Use this argument to set the environment variable **TEST_TARGET_NAME** on all Argo pods. - - * **classname** - testgrid uses **classname** as the test target and allows results to be grouped by name - - * **recommendation** - Set the classname to the workflow name as defined in **prow_config.yaml** - - * This allows easy grouping of tests by the entries defined in **prow_config.yaml** - - * Each entry in **prow_config.yaml** usually corresponds to a different configuration e.g. "GCP with IAP" vs. "GCP with basic auth" - - * So worflow name is a natural grouping - - ### Prow Variables * For each test run PROW defines several variables that pass useful information to your job. diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 5ce5e89fd..000000000 --- a/package-lock.json +++ /dev/null @@ -1,466 +0,0 @@ -{ - "requires": true, - "lockfileVersion": 1, - "dependencies": { - "@textlint/ast-node-types": { - "version": "4.2.5", - "resolved": "https://registry.npmjs.org/@textlint/ast-node-types/-/ast-node-types-4.2.5.tgz", - "integrity": "sha512-+rEx4jLOeZpUcdvll7jEg/7hNbwYvHWFy4IGW/tk2JdbyB3SJVyIP6arAwzTH/sp/pO9jftfyZnRj4//sLbLvQ==" - }, - "@textlint/markdown-to-ast": { - "version": "6.0.9", - "resolved": "https://registry.npmjs.org/@textlint/markdown-to-ast/-/markdown-to-ast-6.0.9.tgz", - "integrity": "sha512-hfAWBvTeUGh5t5kTn2U3uP3qOSM1BSrxzl1jF3nn0ywfZXpRBZr5yRjXnl4DzIYawCtZOshmRi/tI3/x4TE1jQ==", - "requires": { - "@textlint/ast-node-types": "4.2.5", - "debug": "2.6.9", - "remark-frontmatter": "1.3.2", - "remark-parse": "5.0.0", - "structured-source": "3.0.2", - "traverse": "0.6.6", - "unified": "6.2.0" - } - }, - "anchor-markdown-header": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/anchor-markdown-header/-/anchor-markdown-header-0.5.7.tgz", - "integrity": "sha1-BFBj125qH5zTJ6V6ASaqD97Dcac=", - "requires": { - "emoji-regex": "6.1.3" - } - }, - "bail": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/bail/-/bail-1.0.4.tgz", - "integrity": "sha512-S8vuDB4w6YpRhICUDET3guPlQpaJl7od94tpZ0Fvnyp+MKW/HyDTcRDck+29C9g+d/qQHnddRH3+94kZdrW0Ww==" - }, - "boundary": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/boundary/-/boundary-1.0.1.tgz", - "integrity": "sha1-TWfcJgLAzBbdm85+v4fpSCkPWBI=" - }, - "character-entities": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.3.tgz", - "integrity": "sha512-yB4oYSAa9yLcGyTbB4ItFwHw43QHdH129IJ5R+WvxOkWlyFnR5FAaBNnUq4mcxsTVZGh28bHoeTHMKXH1wZf3w==" - }, - "character-entities-legacy": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.3.tgz", - "integrity": "sha512-YAxUpPoPwxYFsslbdKkhrGnXAtXoHNgYjlBM3WMXkWGTl5RsY3QmOyhwAgL8Nxm9l5LBThXGawxKPn68y6/fww==" - }, - "character-reference-invalid": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.3.tgz", - "integrity": "sha512-VOq6PRzQBam/8Jm6XBGk2fNEnHXAdGd6go0rtd4weAGECBamHDwwCQSOT12TACIYUZegUXnV6xBXqUssijtxIg==" - }, - "collapse-white-space": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-1.0.5.tgz", - "integrity": "sha512-703bOOmytCYAX9cXYqoikYIx6twmFCXsnzRQheBcTG3nzKYBR4P/+wkYeH+Mvj7qUz8zZDtdyzbxfnEi/kYzRQ==" - }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "doctoc": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/doctoc/-/doctoc-1.4.0.tgz", - "integrity": "sha512-8IAq3KdMkxhXCUF+xdZxdJxwuz8N2j25sMgqiu4U4JWluN9tRKMlAalxGASszQjlZaBprdD2YfXpL3VPWUD4eg==", - "requires": { - "@textlint/markdown-to-ast": "6.0.9", - "anchor-markdown-header": "0.5.7", - "htmlparser2": "3.9.2", - "minimist": "1.2.0", - "underscore": "1.8.3", - "update-section": "0.3.3" - } - }, - "dom-serializer": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.2.1.tgz", - "integrity": "sha512-sK3ujri04WyjwQXVoK4PU3y8ula1stq10GJZpqHIUgoGZdsGzAGu65BnU3d08aTVSvO7mGPZUc0wTEDL+qGE0Q==", - "requires": { - "domelementtype": "2.0.1", - "entities": "2.0.0" - }, - "dependencies": { - "domelementtype": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.0.1.tgz", - "integrity": "sha512-5HOHUDsYZWV8FGWN0Njbr/Rn7f/eWSQi1v7+HsUVwXgn8nWWlL64zKDkS0n8ZmQ3mlWOMuXOnR+7Nx/5tMO5AQ==" - }, - "entities": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.0.tgz", - "integrity": "sha512-D9f7V0JSRwIxlRI2mjMqufDrRDnx8p+eEOz7aUM9SuvF8gsBzra0/6tbjl1m8eQHrZlYj6PxqE00hZ1SAIKPLw==" - } - } - }, - "domelementtype": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", - "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" - }, - "domhandler": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", - "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", - "requires": { - "domelementtype": "1.3.1" - } - }, - "domutils": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", - "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", - "requires": { - "dom-serializer": "0.2.1", - "domelementtype": "1.3.1" - } - }, - "emoji-regex": { - "version": "6.1.3", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-6.1.3.tgz", - "integrity": "sha1-7HmjlpsC0uzytyJUJ5v5m8eoOTI=" - }, - "entities": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", - "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" - }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "fault": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.3.tgz", - "integrity": "sha512-sfFuP4X0hzrbGKjAUNXYvNqsZ5F6ohx/dZ9I0KQud/aiZNwg263r5L9yGB0clvXHCkzXh5W3t7RSHchggYIFmA==", - "requires": { - "format": "0.2.2" - } - }, - "format": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz", - "integrity": "sha1-1hcBB+nv3E7TDJ3DkBbflCtctYs=" - }, - "htmlparser2": { - "version": "3.9.2", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz", - "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=", - "requires": { - "domelementtype": "1.3.1", - "domhandler": "2.4.2", - "domutils": "1.7.0", - "entities": "1.1.2", - "inherits": "2.0.4", - "readable-stream": "2.3.6" - } - }, - "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" - }, - "is-alphabetical": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.3.tgz", - "integrity": "sha512-eEMa6MKpHFzw38eKm56iNNi6GJ7lf6aLLio7Kr23sJPAECscgRtZvOBYybejWDQ2bM949Y++61PY+udzj5QMLA==" - }, - "is-alphanumerical": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.3.tgz", - "integrity": "sha512-A1IGAPO5AW9vSh7omxIlOGwIqEvpW/TA+DksVOPM5ODuxKlZS09+TEM1E3275lJqO2oJ38vDpeAL3DCIiHE6eA==", - "requires": { - "is-alphabetical": "1.0.3", - "is-decimal": "1.0.3" - } - }, - "is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" - }, - "is-decimal": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.3.tgz", - "integrity": "sha512-bvLSwoDg2q6Gf+E2LEPiklHZxxiSi3XAh4Mav65mKqTfCO1HM3uBs24TjEH8iJX3bbDdLXKJXBTmGzuTUuAEjQ==" - }, - "is-hexadecimal": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.3.tgz", - "integrity": "sha512-zxQ9//Q3D/34poZf8fiy3m3XVpbQc7ren15iKqrTtLPwkPD/t3Scy9Imp63FujULGxuK0ZlCwoo5xNpktFgbOA==" - }, - "is-plain-obj": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", - "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=" - }, - "is-whitespace-character": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-whitespace-character/-/is-whitespace-character-1.0.3.tgz", - "integrity": "sha512-SNPgMLz9JzPccD3nPctcj8sZlX9DAMJSKH8bP7Z6bohCwuNgX8xbWr1eTAYXX9Vpi/aSn8Y1akL9WgM3t43YNQ==" - }, - "is-word-character": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-word-character/-/is-word-character-1.0.3.tgz", - "integrity": "sha512-0wfcrFgOOOBdgRNT9H33xe6Zi6yhX/uoc4U8NBZGeQQB0ctU1dnlNTyL9JM2646bHDTpsDm1Brb3VPoCIMrd/A==" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "markdown-escapes": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/markdown-escapes/-/markdown-escapes-1.0.3.tgz", - "integrity": "sha512-XUi5HJhhV5R74k8/0H2oCbCiYf/u4cO/rX8tnGkRvrqhsr5BRNU6Mg0yt/8UIx1iIS8220BNJsDb7XnILhLepw==" - }, - "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "parse-entities": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-1.2.2.tgz", - "integrity": "sha512-NzfpbxW/NPrzZ/yYSoQxyqUZMZXIdCfE0OIN4ESsnptHJECoUk3FZktxNuzQf4tjt5UEopnxpYJbvYuxIFDdsg==", - "requires": { - "character-entities": "1.2.3", - "character-entities-legacy": "1.1.3", - "character-reference-invalid": "1.1.3", - "is-alphanumerical": "1.0.3", - "is-decimal": "1.0.3", - "is-hexadecimal": "1.0.3" - } - }, - "process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, - "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.4", - "isarray": "1.0.0", - "process-nextick-args": "2.0.1", - "safe-buffer": "5.1.2", - "string_decoder": "1.1.1", - "util-deprecate": "1.0.2" - } - }, - "remark-frontmatter": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/remark-frontmatter/-/remark-frontmatter-1.3.2.tgz", - "integrity": "sha512-2eayxITZ8rezsXdgcXnYB3iLivohm2V/ZT4Ne8uhua6A4pk6GdLE2ZzJnbnINtD1HRLaTdB7RwF9sgUbMptJZA==", - "requires": { - "fault": "1.0.3", - "xtend": "4.0.2" - } - }, - "remark-parse": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-5.0.0.tgz", - "integrity": "sha512-b3iXszZLH1TLoyUzrATcTQUZrwNl1rE70rVdSruJFlDaJ9z5aMkhrG43Pp68OgfHndL/ADz6V69Zow8cTQu+JA==", - "requires": { - "collapse-white-space": "1.0.5", - "is-alphabetical": "1.0.3", - "is-decimal": "1.0.3", - "is-whitespace-character": "1.0.3", - "is-word-character": "1.0.3", - "markdown-escapes": "1.0.3", - "parse-entities": "1.2.2", - "repeat-string": "1.6.1", - "state-toggle": "1.0.2", - "trim": "0.0.1", - "trim-trailing-lines": "1.1.2", - "unherit": "1.1.2", - "unist-util-remove-position": "1.1.3", - "vfile-location": "2.0.5", - "xtend": "4.0.2" - } - }, - "repeat-string": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", - "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=" - }, - "replace-ext": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/replace-ext/-/replace-ext-1.0.0.tgz", - "integrity": "sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs=" - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "state-toggle": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/state-toggle/-/state-toggle-1.0.2.tgz", - "integrity": "sha512-8LpelPGR0qQM4PnfLiplOQNJcIN1/r2Gy0xKB2zKnIW2YzPMt2sR4I/+gtPjhN7Svh9kw+zqEg2SFwpBO9iNiw==" - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "5.1.2" - } - }, - "structured-source": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/structured-source/-/structured-source-3.0.2.tgz", - "integrity": "sha1-3YAkJeD1PcSm56yjdSkBoczaevU=", - "requires": { - "boundary": "1.0.1" - } - }, - "traverse": { - "version": "0.6.6", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz", - "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=" - }, - "trim": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/trim/-/trim-0.0.1.tgz", - "integrity": "sha1-WFhUf2spB1fulczMZm+1AITEYN0=" - }, - "trim-trailing-lines": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/trim-trailing-lines/-/trim-trailing-lines-1.1.2.tgz", - "integrity": "sha512-MUjYItdrqqj2zpcHFTkMa9WAv4JHTI6gnRQGPFLrt5L9a6tRMiDnIqYl8JBvu2d2Tc3lWJKQwlGCp0K8AvCM+Q==" - }, - "trough": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/trough/-/trough-1.0.4.tgz", - "integrity": "sha512-tdzBRDGWcI1OpPVmChbdSKhvSVurznZ8X36AYURAcl+0o2ldlCY2XPzyXNNxwJwwyIU+rIglTCG4kxtNKBQH7Q==" - }, - "underscore": { - "version": "1.8.3", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", - "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=" - }, - "unherit": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/unherit/-/unherit-1.1.2.tgz", - "integrity": "sha512-W3tMnpaMG7ZY6xe/moK04U9fBhi6wEiCYHUW5Mop/wQHf12+79EQGwxYejNdhEz2mkqkBlGwm7pxmgBKMVUj0w==", - "requires": { - "inherits": "2.0.4", - "xtend": "4.0.2" - } - }, - "unified": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/unified/-/unified-6.2.0.tgz", - "integrity": "sha512-1k+KPhlVtqmG99RaTbAv/usu85fcSRu3wY8X+vnsEhIxNP5VbVIDiXnLqyKIG+UMdyTg0ZX9EI6k2AfjJkHPtA==", - "requires": { - "bail": "1.0.4", - "extend": "3.0.2", - "is-plain-obj": "1.1.0", - "trough": "1.0.4", - "vfile": "2.3.0", - "x-is-string": "0.1.0" - } - }, - "unist-util-is": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-3.0.0.tgz", - "integrity": "sha512-sVZZX3+kspVNmLWBPAB6r+7D9ZgAFPNWm66f7YNb420RlQSbn+n8rG8dGZSkrER7ZIXGQYNm5pqC3v3HopH24A==" - }, - "unist-util-remove-position": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-1.1.3.tgz", - "integrity": "sha512-CtszTlOjP2sBGYc2zcKA/CvNdTdEs3ozbiJ63IPBxh8iZg42SCCb8m04f8z2+V1aSk5a7BxbZKEdoDjadmBkWA==", - "requires": { - "unist-util-visit": "1.4.1" - } - }, - "unist-util-stringify-position": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-1.1.2.tgz", - "integrity": "sha512-pNCVrk64LZv1kElr0N1wPiHEUoXNVFERp+mlTg/s9R5Lwg87f9bM/3sQB99w+N9D/qnM9ar3+AKDBwo/gm/iQQ==" - }, - "unist-util-visit": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", - "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", - "requires": { - "unist-util-visit-parents": "2.1.2" - } - }, - "unist-util-visit-parents": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-2.1.2.tgz", - "integrity": "sha512-DyN5vD4NE3aSeB+PXYNKxzGsfocxp6asDc2XXE3b0ekO2BaRUpBicbbUygfSvYfUz1IkmjFR1YF7dPklraMZ2g==", - "requires": { - "unist-util-is": "3.0.0" - } - }, - "update-section": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/update-section/-/update-section-0.3.3.tgz", - "integrity": "sha1-RY8Xgg03gg3GDiC4bZQ5GwASMVg=" - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "vfile": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/vfile/-/vfile-2.3.0.tgz", - "integrity": "sha512-ASt4mBUHcTpMKD/l5Q+WJXNtshlWxOogYyGYYrg4lt/vuRjC1EFQtlAofL5VmtVNIZJzWYFJjzGWZ0Gw8pzW1w==", - "requires": { - "is-buffer": "1.1.6", - "replace-ext": "1.0.0", - "unist-util-stringify-position": "1.1.2", - "vfile-message": "1.1.1" - } - }, - "vfile-location": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-2.0.5.tgz", - "integrity": "sha512-Pa1ey0OzYBkLPxPZI3d9E+S4BmvfVwNAAXrrqGbwTVXWaX2p9kM1zZ+n35UtVM06shmWKH4RPRN8KI80qE3wNQ==" - }, - "vfile-message": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-1.1.1.tgz", - "integrity": "sha512-1WmsopSGhWt5laNir+633LszXvZ+Z/lxveBf6yhGsqnQIhlhzooZae7zV6YVM1Sdkw68dtAW3ow0pOdPANugvA==", - "requires": { - "unist-util-stringify-position": "1.1.2" - } - }, - "x-is-string": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz", - "integrity": "sha1-R0tQhlrzpJqcRlfwWs0UVFj3fYI=" - }, - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" - } - } -} diff --git a/py/kubeflow/testing/argo_build_util.py b/py/kubeflow/testing/argo_build_util.py index 776a2f3e8..e265a418a 100644 --- a/py/kubeflow/testing/argo_build_util.py +++ b/py/kubeflow/testing/argo_build_util.py @@ -150,7 +150,6 @@ def add_task_to_dag(workflow, dag_name, task, dependencies): workflow["spec"]["templates"].append(new_template) - return new_template def set_task_template_labels(workflow): """Automatically set the labels and annotations on each step. diff --git a/py/kubeflow/testing/ci/kf_unittests.py b/py/kubeflow/testing/ci/kf_unittests.py index 5947c2f05..72905e316 100644 --- a/py/kubeflow/testing/ci/kf_unittests.py +++ b/py/kubeflow/testing/ci/kf_unittests.py @@ -15,13 +15,10 @@ TEMPLATE_LABEL = "kf_unittests" class Builder: # pylint: disable=too-many-instance-attributes - def __init__(self, name=None, namespace=None, bucket=None, - test_target_name=None, - **kwargs): # pylint: disable=unused-argument + def __init__(self, name=None, namespace=None, bucket=None): self.name = name self.namespace = namespace self.bucket = bucket - self.test_target_name = test_target_name #**************************************************************************** # Define directory locations @@ -34,8 +31,7 @@ def __init__(self, name=None, namespace=None, bucket=None, # output_dir is the directory to sync to GCS to contain the output for this # job. self.output_dir = self.test_dir + "/output" - - self.artifacts_dir = self.output_dir + "/artifacts/junit_{0}".format(name) + self.artifacts_dir = self.output_dir + "/artifacts" # source directory where all repos should be checked out self.src_root_dir = self.test_dir + "/src" @@ -139,12 +135,6 @@ def _build_task_template(self): task_template["container"]["env"].extend(common_env) - if self.test_target_name: - task_template["container"]["env"].append({ - 'name': 'TEST_TARGET_NAME', - 'value': self.test_target_name, - }) - task_template = argo_build_util.add_prow_env(task_template) return task_template @@ -173,22 +163,6 @@ def build(self): argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, checkout, []) - #************************************************************************** - # Make dir - # pytest was failing trying to call makedirs. My suspicion is its - # because the two steps ended up trying to create the directory at the - # same time and classing. So we create a separate step to do it. - mkdir_step = argo_build_util.deep_copy(task_template) - - mkdir_step["name"] = "make-artifacts-dir" - mkdir_step["container"]["command"] = ["mkdir", - "-p", - self.artifacts_dir] - - - argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, mkdir_step, - [checkout["name"]]) - #************************************************************************** # Run python unittests py_tests = argo_build_util.deep_copy(task_template) @@ -205,7 +179,7 @@ def build(self): argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, py_tests, - [mkdir_step["name"]]) + [checkout["name"]]) #*************************************************************************** @@ -214,27 +188,17 @@ def build(self): py_lint = argo_build_util.deep_copy(task_template) py_lint["name"] = "py-lint" - py_lint["container"]["command"] = ["pytest", - "test_py_lint.py", - # I think -s mean stdout/stderr will - # print out to aid in debugging. - # Failures still appear to be captured - # and stored in the junit file. - "-s", + py_lint["container"]["command"] = ["python", + "-m", + "kubeflow.testing.test_py_lint", + "--artifacts_dir=" + self.artifacts_dir, "--src_dir=" + self.kubeflow_testing_py, "--rcfile=" + os.path.join( self.testing_src_dir, ".pylintrc"), - # Test timeout in seconds. - "--timeout=500", - "--junitxml=" + self.artifacts_dir + - "/junit_py-lint.xml"] - - py_lint_step = argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, - py_lint, - [mkdir_step["name"]]) + ] - py_lint_step["container"]["workingDir"] = os.path.join( - self.testing_src_dir, "py/kubeflow/testing") + argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, py_lint, + [checkout["name"]]) #***************************************************************************** # create_pr_symlink @@ -281,7 +245,7 @@ def build(self): return workflow -def create_workflow(name=None, namespace=None, bucket=None, **kwargs): # pylint: disable=too-many-statements +def create_workflow(name=None, namespace=None, bucket=None): # pylint: disable=too-many-statements """Create workflow returns an Argo workflow to test kfctl upgrades. Args: @@ -289,6 +253,6 @@ def create_workflow(name=None, namespace=None, bucket=None, **kwargs): # pylint: associated with the workflow. """ - builder = Builder(name=name, namespace=namespace, bucket=bucket, **kwargs) + builder = Builder(name=name, namespace=namespace, bucket=bucket) return builder.build() diff --git a/py/kubeflow/testing/conftest.py b/py/kubeflow/testing/conftest.py deleted file mode 100644 index 4380526a0..000000000 --- a/py/kubeflow/testing/conftest.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import pytest - -def pytest_addoption(parser): - parser.addoption( - "--src_dir", - action="store", - default=os.getcwd(), - help=("The root directory of the source tree. Defaults to current " - "directory.")) - - parser.addoption( - "--rcfile", - default="", - action="store", - help=("Path to the rcfile.")) - -@pytest.fixture -def rcfile(request): - return request.config.getoption("--rcfile") - -@pytest.fixture -def src_dir(request): - return request.config.getoption("--src_dir") diff --git a/py/kubeflow/testing/prow_artifacts.py b/py/kubeflow/testing/prow_artifacts.py index 7d40ec287..09683fdde 100644 --- a/py/kubeflow/testing/prow_artifacts.py +++ b/py/kubeflow/testing/prow_artifacts.py @@ -199,6 +199,12 @@ def create_pr_symlink(args): blob = bucket.blob(path) blob.upload_from_string(target) +def _get_actual_junit_files(bucket, prefix): + actual_junit = set() + for b in bucket.list_blobs(prefix=os.path.join(prefix, "junit")): + actual_junit.add(os.path.basename(b.name)) + return actual_junit + def check_no_errors(gcs_client, artifacts_dir): """Check that all the XML files exist and there were no errors. Args: @@ -211,12 +217,14 @@ def check_no_errors(gcs_client, artifacts_dir): bucket = gcs_client.get_bucket(bucket_name) no_errors = True - for b in bucket.list_blobs(prefix=os.path.join(prefix, "junit")): - full_path = util.to_gcs_uri(b.bucket, b.path) - if not os.path.splitext(b.path)[-1] == ".xml": - logging.info("Skipping %s; not an xml file", full_path) - continue + # Get a list of actual junit files. + actual_junit = _get_actual_junit_files(bucket, prefix) + + for f in actual_junit: + full_path = os.path.join(artifacts_dir, f) logging.info("Checking %s", full_path) + b = bucket.blob(os.path.join(prefix, f)) + xml_contents = b.download_as_string() if test_util.get_num_failures(xml_contents) > 0: diff --git a/py/kubeflow/testing/run_e2e_workflow.py b/py/kubeflow/testing/run_e2e_workflow.py index 466253373..a63371a36 100644 --- a/py/kubeflow/testing/run_e2e_workflow.py +++ b/py/kubeflow/testing/run_e2e_workflow.py @@ -69,13 +69,6 @@ import sys import yaml -# The name of the command line argument for workflows for the var -# to contain the test target name. -# The goal is to be able to use target name grouping in test grid -# to group related tests -# https://github.com/kubernetes/test-infra/tree/master/testgrid#grouping-tests -TEST_TARGET_ARG_NAME = "test_target_name" - # The namespace to launch the Argo workflow in. def get_namespace(args): if args.namespace: @@ -336,12 +329,6 @@ def run(args, file_handler): # pylint: disable=too-many-statements,too-many-bran w.kwargs["name"] = workflow_name w.kwargs["namespace"] = get_namespace(args) - if TEST_TARGET_ARG_NAME not in w.kwargs: - w.kwargs[TEST_TARGET_ARG_NAME] = w.name - logging.info("Workflow %s doesn't set arg %s; defaulting to %s", - w.name, TEST_TARGET_ARG_NAME, - w.kwargs[TEST_TARGET_ARG_NAME]) - # TODO(https://github.com/kubeflow/testing/issues/467): We shell out # to e2e_tool in order to dumpy the Argo workflow to a file which then # reimport. We do this because importing the py_func module appears diff --git a/py/kubeflow/testing/test_py_lint.py b/py/kubeflow/testing/test_py_lint.py index e69120beb..c18872e1e 100644 --- a/py/kubeflow/testing/test_py_lint.py +++ b/py/kubeflow/testing/test_py_lint.py @@ -1,19 +1,11 @@ +import argparse import fnmatch import logging import os import subprocess -from kubeflow.testing import util +from kubeflow.testing import test_helper, util -import pytest - -logging.basicConfig( - level=logging.INFO, - format=('%(levelname)s|%(asctime)s' - '|%(pathname)s|%(lineno)d| %(message)s'), - datefmt='%Y-%m-%dT%H:%M:%S', -) -logging.getLogger().setLevel(logging.INFO) def should_exclude(root, full_dir_excludes): for e in full_dir_excludes: @@ -22,13 +14,27 @@ def should_exclude(root, full_dir_excludes): return False -def test_lint(record_xml_attribute, src_dir, rcfile): # pylint: disable=redefined-outer-name - # Override the classname attribute in the junit file. - # This makes it easy to group related tests in test grid. - # http://doc.pytest.org/en/latest/usage.html#record-xml-attribute - util.set_pytest_junit(record_xml_attribute, "test_py_lint") +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--src_dir", + default=os.getcwd(), + type=str, + help=("The root directory of the source tree. Defaults to current " + "directory.")) + + parser.add_argument( + "--rcfile", + default="", + type=str, + help=("Path to the rcfile.")) + args, _ = parser.parse_known_args() + return args + +def test_lint(test_case): # pylint: disable=redefined-outer-name logging.info('Running test_lint') + args = parse_args() # Print out the pylint version because different versions can produce # different results. util.run(["pylint", "--version"]) @@ -42,16 +48,18 @@ def test_lint(record_xml_attribute, src_dir, rcfile): # pylint: disable=redefine "release-infra", ] full_dir_excludes = [ - os.path.join(os.path.abspath(src_dir), f) for f in dir_excludes + os.path.join(os.path.abspath(args.src_dir), f) for f in dir_excludes ] # TODO(jlewi): Use pathlib once we switch to python3. includes = ["*.py"] failed_files = [] - if not rcfile: - rcfile = os.path.join(src_dir, ".pylintrc") + if not args.rcfile: + rc_file = os.path.join(args.src_dir, ".pylintrc") + else: + rc_file = args.rcfile - for root, dirs, files in os.walk(os.path.abspath(src_dir), topdown=True): + for root, dirs, files in os.walk(os.path.abspath(args.src_dir), topdown=True): # Exclude vendor directories and all sub files. if "vendor" in root.split(os.sep): continue @@ -67,24 +75,20 @@ def test_lint(record_xml_attribute, src_dir, rcfile): # pylint: disable=redefine full_path = os.path.join(root, f) try: util.run( - ["pylint", "--rcfile=" + rcfile, full_path], cwd=src_dir) + ["pylint", "--rcfile=" + rc_file, full_path], cwd=args.src_dir) except subprocess.CalledProcessError: - failed_files.append(full_path[len(src_dir):]) + failed_files.append(full_path[len(args.src_dir):]) if failed_files: failed_files.sort() + test_case.add_failure_info("Files with lint issues: {0}".format( + ", ".join(failed_files))) logging.error("%s files had lint errors:\n%s", len(failed_files), "\n".join(failed_files)) else: logging.info("No lint issues.") - assert not failed_files if __name__ == "__main__": - logging.basicConfig( - level=logging.INFO, - format=('%(levelname)s|%(asctime)s' - '|%(pathname)s|%(lineno)d| %(message)s'), - datefmt='%Y-%m-%dT%H:%M:%S', - ) - logging.getLogger().setLevel(logging.INFO) - pytest.main() + test_case = test_helper.TestCase(name='test_lint', test_func=test_lint) + test_suite = test_helper.init(name='py_lint', test_cases=[test_case]) + test_suite.run() diff --git a/py/kubeflow/testing/util.py b/py/kubeflow/testing/util.py index c75e32de7..ad9ad2cd0 100755 --- a/py/kubeflow/testing/util.py +++ b/py/kubeflow/testing/util.py @@ -1,4 +1,6 @@ """Utilities used by our python scripts for building and releasing.""" +from __future__ import print_function + import datetime import logging import multiprocessing @@ -706,37 +708,3 @@ class JobTimeoutError(TimeoutError): def __init__(self, message, job): super(JobTimeoutError, self).__init__(message) self.job = job - -def set_pytest_junit(record_xml_attribute, test_name): - """Set various xml attributes in the junit produced by pytest. - - pytest supports setting various XML attributes in the junit file. - http://doc.pytest.org/en/latest/usage.html#record-xml-attribute - - test grid supports grouping based on these attributes - https://github.com/kubernetes/test-infra/tree/master/testgrid#grouping-tests - - The goal of this function is to set these attributes in a consistent fashion - to allow easy grouping of tests that were run as part of the same workflow. - """ - # Look for an environment variable named test target name. - TARGET_ENV_NAME = "TEST_TARGET_NAME" - test_target_name = os.getenv(TARGET_ENV_NAME) - full_test_name = test_name - if test_target_name: - # Override the classname attribute in the junit file. - # This makes it easy to group related tests in test grid. - # http://doc.pytest.org/en/latest/usage.html#record-xml-attribute - # Its currently unclear whether testgrid uses classname or testsuite - # Based on the code it looks like its using testsuite name but it - # it doesn't look like we can set that using pytest - record_xml_attribute("classname", test_target_name) - - # Test grid supports grouping into a hierarchy based on the test name. - # To support that we set the test name to include target name. - full_test_name = "/".join([test_target_name, test_name]) - else: - logging.info("Environment variable %s not set; no target name set.", - TARGET_ENV_NAME) - - record_xml_attribute("name", full_test_name)