From 39ef3dba1a6d8b67e1a9e77064714e8a769063df Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 14 Dec 2019 16:57:59 +0000 Subject: [PATCH 01/11] add snap tests - fixes #2954 --- .travis.yml | 2 +- scripts/build_package.sh | 27 +++++++++++++-------------- scripts/build_snap.sh | 27 +++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 15 deletions(-) create mode 100755 scripts/build_snap.sh diff --git a/.travis.yml b/.travis.yml index 9aa2cca1fc..0bf07cd4fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,7 @@ jobs: - sudo /snap/bin/lxd init --auto install: script: - - sudo snapcraft --use-lxd + - ./scripts/build_snap.sh after_failure: - sudo journalctl -u snapd diff --git a/scripts/build_package.sh b/scripts/build_package.sh index 1b95366974..e65e4d75bf 100755 --- a/scripts/build_package.sh +++ b/scripts/build_package.sh @@ -1,11 +1,10 @@ #!/bin/bash - set -e set -x if [ ! -d "dvc" ]; then - echo "Please run this script from repository root" - exit 1 + echo "Please run this script from repository root" + exit 1 fi echo 'PKG = "pip"' > dvc/utils/build.py @@ -15,15 +14,15 @@ python setup.py bdist_wheel --universal # Make sure we have a correct version if [[ -n "$TRAVIS_TAG" ]]; then - pip uninstall -y dvc - if which dvc; then - echo "ERROR: dvc command still exists! Unable to verify dvc version." - exit 1 - fi - pip install dist/dvc-*.whl - if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then - echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" - exit 1 - fi - pip uninstall -y dvc + pip uninstall -y dvc + if which dvc; then + echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 + exit 1 + fi + pip install dist/dvc-*.whl + if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then + echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 + exit 1 + fi + pip uninstall -y dvc fi diff --git a/scripts/build_snap.sh b/scripts/build_snap.sh new file mode 100755 index 0000000000..c6208d1d50 --- /dev/null +++ b/scripts/build_snap.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e +set -x + +if [ ! -d "dvc" ]; then + echo "Please run this script from repository root" + exit 1 +fi + +sudo snapcraft --use-lxd + +# Make sure we have a correct version +if [[ -n "$TRAVIS_TAG" ]]; then + pip uninstall -y dvc + if which dvc; then + echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 + exit 1 + fi + sudo snap install --dangerous --classic dvc_*.snap + if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then + echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 + exit 1 + fi + dvc version + dvc get https://github.com/iterative/dvc scripts/innosetup/dvc.ico + sudo snap remove dvc +fi From 46e06166140b49e902b131cd52ac36d605747ddc Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sat, 14 Dec 2019 17:12:15 +0000 Subject: [PATCH 02/11] promote snap tests for debugging --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0bf07cd4fa..5a5b807938 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,6 +74,7 @@ jobs: python: 3.7 script: ./scripts/build_package.sh - name: Snapcraft snap + stage: check # TODO: change to 'deploy' before merge addons: snaps: - name: snapcraft From b756f77787c3ad4ad8a8c5281df999d112be3684 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 17 Dec 2019 18:57:26 +0000 Subject: [PATCH 03/11] revert whitespace changes, but keep stderr errors --- scripts/build_package.sh | 27 ++++++++++++++------------- scripts/build_posix.sh | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/scripts/build_package.sh b/scripts/build_package.sh index e65e4d75bf..1266ead8bf 100755 --- a/scripts/build_package.sh +++ b/scripts/build_package.sh @@ -1,10 +1,11 @@ #!/bin/bash + set -e set -x if [ ! -d "dvc" ]; then - echo "Please run this script from repository root" - exit 1 + echo "Please run this script from repository root" >&2 + exit 1 fi echo 'PKG = "pip"' > dvc/utils/build.py @@ -14,15 +15,15 @@ python setup.py bdist_wheel --universal # Make sure we have a correct version if [[ -n "$TRAVIS_TAG" ]]; then - pip uninstall -y dvc - if which dvc; then - echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 - exit 1 - fi - pip install dist/dvc-*.whl - if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then - echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 - exit 1 - fi - pip uninstall -y dvc + pip uninstall -y dvc + if which dvc; then + echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 + exit 1 + fi + pip install dist/dvc-*.whl + if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then + echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 + exit 1 + fi + pip uninstall -y dvc fi diff --git a/scripts/build_posix.sh b/scripts/build_posix.sh index c5e9bb89f2..a3deebdc2d 100755 --- a/scripts/build_posix.sh +++ b/scripts/build_posix.sh @@ -29,7 +29,7 @@ fi print_error() { - echo -e "\e[31m$1\e[0m" + echo -e "\e[31m$1\e[0m" >&2 } if [ ! -d "dvc" ]; then From d020c3d8b025204bf9b81a82331df31de00f1c2d Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Tue, 17 Dec 2019 19:10:26 +0000 Subject: [PATCH 04/11] test snaps even where tag is undefined --- scripts/build_snap.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/build_snap.sh b/scripts/build_snap.sh index c6208d1d50..f0c0c61c29 100755 --- a/scripts/build_snap.sh +++ b/scripts/build_snap.sh @@ -9,19 +9,20 @@ fi sudo snapcraft --use-lxd -# Make sure we have a correct version +pip uninstall -y dvc +if which dvc; then + echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 + exit 1 +fi +sudo snap install --dangerous --classic dvc_*.snap if [[ -n "$TRAVIS_TAG" ]]; then - pip uninstall -y dvc - if which dvc; then - echo "ERROR: dvc command still exists! Unable to verify dvc version." >&2 - exit 1 - fi - sudo snap install --dangerous --classic dvc_*.snap + # Make sure we have a correct version if [[ "$(dvc --version)" != "$TRAVIS_TAG" ]]; then echo "ERROR: 'dvc --version'$(dvc -V) doesn't match '$TRAVIS_TAG'" >&2 exit 1 fi - dvc version - dvc get https://github.com/iterative/dvc scripts/innosetup/dvc.ico - sudo snap remove dvc fi +# ensure basic commands can run +dvc version +dvc get https://github.com/iterative/dvc scripts/innosetup/dvc.ico +sudo snap remove dvc From 0a0b4e019e0dee5fbc7b149a6d1bead542ff107c Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 18 Dec 2019 14:17:03 +0000 Subject: [PATCH 05/11] debugging --- scripts/build_snap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_snap.sh b/scripts/build_snap.sh index f0c0c61c29..4631d8601a 100755 --- a/scripts/build_snap.sh +++ b/scripts/build_snap.sh @@ -24,5 +24,5 @@ if [[ -n "$TRAVIS_TAG" ]]; then fi # ensure basic commands can run dvc version -dvc get https://github.com/iterative/dvc scripts/innosetup/dvc.ico +dvc get -v https://github.com/iterative/dvc scripts/innosetup/dvc.ico sudo snap remove dvc From 5ac849377ac79681861069716e0eb8dbeab6b024 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 8 Jan 2020 12:38:08 +0530 Subject: [PATCH 06/11] minor comment update --- snap/snapcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b1e9ed0588..e73da698ef 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -3,7 +3,7 @@ summary: Data Version Control description: Git for Data & Models https://dvc.org adopt-info: dvc grade: stable -confinement: classic # for same reasons as snap `git-ubuntu` +confinement: classic # for `dvc run`ning external binaries base: core18 license: Apache-2.0 layout: From 2fe9b3a967e03cb174c32dfdac9bbb3645f87f31 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Wed, 8 Jan 2020 12:38:22 +0530 Subject: [PATCH 07/11] connectivity debugging --- scripts/build_snap.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_snap.sh b/scripts/build_snap.sh index 4631d8601a..9f1b86b7bd 100755 --- a/scripts/build_snap.sh +++ b/scripts/build_snap.sh @@ -24,5 +24,7 @@ if [[ -n "$TRAVIS_TAG" ]]; then fi # ensure basic commands can run dvc version +dvc.git status +dvc.git fetch --all dvc get -v https://github.com/iterative/dvc scripts/innosetup/dvc.ico sudo snap remove dvc From 205077e3a0a0dd1c04df7e4a0ea9fe6f31010036 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 10 Jan 2020 14:05:57 +0530 Subject: [PATCH 08/11] debugging --- dvc/repo/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 9cef82e11f..037941144f 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -130,10 +130,13 @@ def __repr__(self): @classmethod def find_root(cls, root=None): + logger.info("find_root") + logger.info(root) if root is None: root = os.getcwd() else: root = os.path.abspath(os.path.realpath(root)) + logger.info(root) while True: dvc_dir = os.path.join(root, cls.DVC_DIR) @@ -142,6 +145,7 @@ def find_root(cls, root=None): if os.path.ismount(root): break root = os.path.dirname(root) + logger.info(root) raise NotDvcRepoError(root) @classmethod From 22ed9cb5dc8e8f3a9e792311718db613552eeeb0 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 19 Jan 2020 18:22:39 +0000 Subject: [PATCH 09/11] Remove travis-incompatible tests - reverts + 205077e3a0a0dd1c04df7e4a0ea9fe6f31010036. + 46e06166140b49e902b131cd52ac36d605747ddc. --- .travis.yml | 1 - dvc/repo/__init__.py | 4 ---- scripts/build_snap.sh | 2 +- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5a5b807938..0bf07cd4fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,7 +74,6 @@ jobs: python: 3.7 script: ./scripts/build_package.sh - name: Snapcraft snap - stage: check # TODO: change to 'deploy' before merge addons: snaps: - name: snapcraft diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 037941144f..9cef82e11f 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -130,13 +130,10 @@ def __repr__(self): @classmethod def find_root(cls, root=None): - logger.info("find_root") - logger.info(root) if root is None: root = os.getcwd() else: root = os.path.abspath(os.path.realpath(root)) - logger.info(root) while True: dvc_dir = os.path.join(root, cls.DVC_DIR) @@ -145,7 +142,6 @@ def find_root(cls, root=None): if os.path.ismount(root): break root = os.path.dirname(root) - logger.info(root) raise NotDvcRepoError(root) @classmethod diff --git a/scripts/build_snap.sh b/scripts/build_snap.sh index 9f1b86b7bd..ce78aea583 100755 --- a/scripts/build_snap.sh +++ b/scripts/build_snap.sh @@ -23,8 +23,8 @@ if [[ -n "$TRAVIS_TAG" ]]; then fi fi # ensure basic commands can run +# N.B.: cannot run `dvc get` on travis (#2956) dvc version dvc.git status dvc.git fetch --all -dvc get -v https://github.com/iterative/dvc scripts/innosetup/dvc.ico sudo snap remove dvc From de358d661ad6e027eed4f151e5de4cca92402760 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 19 Jan 2020 19:02:29 +0000 Subject: [PATCH 10/11] explicitly spilt snap stable/edge deployments - (re) fix #3152 --- .travis.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0bf07cd4fa..14451410af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -135,10 +135,19 @@ deploy: stage: build - provider: snap snap: dvc_*.snap - channel: ${TRAVIS_TAG:+stable}${TRAVIS_TAG:-edge} + channel: stable skip_cleanup: true on: - all_branches: true - condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)" + condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap)" + tags: true + repo: iterative/dvc + stage: build + - provider: snap + snap: dvc_*.snap + channel: edge + skip_cleanup: true + on: + branch: master + condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap)" repo: iterative/dvc stage: build From f80626b5d0acab92f42ef471b4f4602dc8711743 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 19 Jan 2020 19:09:47 +0000 Subject: [PATCH 11/11] recompress conditional snap channel --- .travis.yml | 15 +++------------ scripts/ci/before_install.sh | 6 ++++++ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 14451410af..1bb1d86079 100644 --- a/.travis.yml +++ b/.travis.yml @@ -135,19 +135,10 @@ deploy: stage: build - provider: snap snap: dvc_*.snap - channel: stable + channel: $SNAP_CHANNEL skip_cleanup: true on: - condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap)" - tags: true - repo: iterative/dvc - stage: build - - provider: snap - snap: dvc_*.snap - channel: edge - skip_cleanup: true - on: - branch: master - condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap)" + all_branches: true + condition: "$(./scripts/ci/deploy_condition.sh dvc_*.snap) && ($TRAVIS_BRANCH = master || -n $TRAVIS_TAG)" repo: iterative/dvc stage: build diff --git a/scripts/ci/before_install.sh b/scripts/ci/before_install.sh index 27c423b7d2..2e9461be41 100644 --- a/scripts/ci/before_install.sh +++ b/scripts/ci/before_install.sh @@ -50,3 +50,9 @@ elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then ln -s -f /usr/local/bin/python3 /usr/local/bin/python ln -s -f /usr/local/bin/pip3 /usr/local/bin/pip fi + +if [[ -n "$TRAVIS_TAG" ]]; then + echo "export SNAP_CHANNEL=stable" >> env.sh +else + echo "export SNAP_CHANNEL=edge" >> env.sh +fi