This repository has been archived by the owner on Sep 1, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 276
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
DEVOPS-383: Move deploy to S3 job to circleci workflow
- Loading branch information
1 parent
a05a512
commit 64fb803
Showing
3 changed files
with
122 additions
and
156 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,133 +1,142 @@ | ||
| version: 2 | ||
| jobs: | ||
| build: | ||
| build-and-test: | ||
| macos: | ||
| xcode: '9.0.1' | ||
| working_directory: ~/numenta/nupic.core | ||
| parallelism: 1 | ||
| shell: /bin/bash --login -o pipefail | ||
| # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did. | ||
| # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables . | ||
| environment: | ||
| CIRCLE_ARTIFACTS: /tmp/circleci-artifacts | ||
| CIRCLE_TEST_REPORTS: /tmp/circleci-test-results | ||
| XCODE_SCHEME: nupic | ||
| XCODE_WORKSPACE: nupic | ||
| ARCHFLAGS: -arch x86_64 | ||
| PYTHONPATH: ~/Library/Python/2.7/lib/python/site-packages | ||
| PYBIN: ~/Library/Python/2.7/bin | ||
| macos: | ||
| xcode: '9.0.1' | ||
| steps: | ||
| # Machine Setup | ||
| # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each | ||
| - run: sudo systemsetup -settimezone 'GMT' | ||
| # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out. | ||
| - checkout | ||
| # Prepare for artifact and test results collection equivalent to how it was done on 1.0. | ||
| # In many cases you can simplify this from what is generated here. | ||
| # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/' | ||
| - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS | ||
| - run: | ||
| name: Update PATH and Define Environment Variable at Runtime | ||
| command: echo 'export PATH=$HOME/Library/Python/2.7/bin:$PATH' >> $BASH_ENV | ||
| - run: | ||
| name: Make sure to use OS X in CircleCI Web UI | ||
| command: | | ||
| if [[ "$OSTYPE" != "darwin"* ]]; then | ||
| echo "Must set option to use OS X in CircleCI Web UI" && exit 1; | ||
| fi | ||
| # Dependencies | ||
| # This would typically go in either a build or a build-and-test job when using workflows | ||
| # Restore the dependency cache | ||
| - restore_cache: | ||
| keys: | ||
| # This branch if available | ||
| - v1-dep-{{ .Branch }}- | ||
| # Default branch if not | ||
| - v1-dep-master- | ||
| # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly | ||
| - v1-dep- | ||
| # Machine Setup | ||
| - run: | ||
| name: Make sure to use OS X in CircleCI Web UI | ||
| command: | | ||
| if [[ "$OSTYPE" != "darwin"* ]]; then | ||
| echo "Must set option to use OS X in CircleCI Web UI" && exit 1; | ||
| fi | ||
| - run: sudo systemsetup -settimezone 'GMT' | ||
| - run: | ||
| name: Restoring system python | ||
| command: | | ||
| brew uninstall python | ||
| curl https://bootstrap.pypa.io/get-pip.py | python - --user | ||
| echo 'export PATH=$HOME/Library/Python/2.7/bin:$PATH' >> $BASH_ENV | ||
| - run: | ||
| name: Installing cmake | ||
| command: brew install cmake | ||
|
|
||
| - checkout | ||
|
|
||
| - run: | ||
| name: Installing dependencies | ||
| command: | | ||
| brew install cmake | ||
| pip install --user --upgrade --verbose setuptools setuptools-scm | ||
| pip install --no-cache-dir --user -r bindings/py/requirements.txt --verbose || exit | ||
| # Dependencies | ||
| # Restore the dependency cache | ||
| - restore_cache: | ||
| keys: | ||
| # This branch if available | ||
| - v1-dep-{{ .Branch }}- | ||
| # Default branch if not | ||
| - v1-dep-master- | ||
| # Any branch if there are none on the default branch - this should be | ||
| # unnecessary if you have your default branch configured correctly | ||
| - v1-dep- | ||
|
|
||
| # Save dependency cache | ||
| - save_cache: | ||
| key: v1-dep-{{ .Branch }}-{{ epoch }} | ||
| paths: | ||
| # This is a broad list of cache paths to include many possible development environments | ||
| # You can probably delete some of these entries | ||
| - vendor/bundle | ||
| - ~/virtualenvs | ||
| - ~/.m2 | ||
| - ~/.ivy2 | ||
| - ~/.bundle | ||
| - ~/.go_workspace | ||
| - ~/.gradle | ||
| - ~/.cache/bower | ||
| # Compile | ||
| # This would typically go in either a build or a build-and-test job when using workflows | ||
| - run: | ||
| name: Compiling | ||
| environment: | ||
| VERBOSE: 1 | ||
| command: | | ||
| mkdir -p build/scripts | ||
| cd build/scripts | ||
| cmake ../.. -DCMAKE_BUILD_TYPE=Release -DNTA_COV_ENABLED=ON -DCMAKE_INSTALL_PREFIX=../release -DPY_EXTENSIONS_DIR=../../bindings/py/src/nupic/bindings | ||
| make -j8 | grep -v -F '\\-\\- Installing:' | ||
| make install 2>&1 | grep -v -F 'Installing:' | ||
| - run: | ||
| name: Build wheel distribution | ||
| command: | | ||
| python setup.py bdist_wheel | ||
| mkdir dist | ||
| mv bindings/py/dist/*.whl dist/ | ||
| # Test | ||
| # This would typically be a build job when using workflows, possibly combined with build | ||
| - run: | ||
| name: Running C++ Tests | ||
| working_directory: ~/numenta/nupic.core/build/release/bin | ||
| command: | | ||
| - run: | ||
| name: Installing dependencies | ||
| command: | | ||
| pip install --user --upgrade --verbose pip setuptools setuptools-scm | ||
| pip install --no-cache-dir --user -r bindings/py/requirements.txt --verbose || exit | ||
| # Save dependency cache | ||
| - save_cache: | ||
| key: v1-dep-{{ .Branch }}-{{ epoch }} | ||
| paths: | ||
| # This is a broad list of cache paths to include many possible | ||
| # development environments. | ||
| - vendor/bundle | ||
| - ~/virtualenvs | ||
| - ~/.m2 | ||
| - ~/.ivy2 | ||
| - ~/.bundle | ||
| - ~/.go_workspace | ||
| - ~/.gradle | ||
| - ~/.cache/bower | ||
|
|
||
| # Build | ||
| - run: | ||
| name: Compiling | ||
| environment: | ||
| VERBOSE: 1 | ||
| command: | | ||
| mkdir -p build/scripts | ||
| cd build/scripts | ||
| cmake ../.. -DCMAKE_BUILD_TYPE=Release -DNTA_COV_ENABLED=ON -DCMAKE_INSTALL_PREFIX=../release -DPY_EXTENSIONS_DIR=../../bindings/py/src/nupic/bindings | ||
| make | grep -v -F '\\-\\- Installing:' | ||
| make install 2>&1 | grep -v -F 'Installing:' | ||
| - run: | ||
| name: Build distribution | ||
| command: | | ||
| python setup.py bdist_wheel | ||
| mkdir -p dist/include/nupic | ||
| mv bindings/py/dist/*.whl dist/ | ||
| cp bindings/py/requirements.txt dist/ | ||
| cp build/release/include/nupic/Version.hpp dist/include/nupic/ | ||
| # Test | ||
| - run: | ||
| name: Running python tests | ||
| command: | | ||
| mkdir -p tests/py | ||
| pip install --user --no-index --find-links=$CIRCLE_WORKING_DIRECTORY/dist/ nupic.bindings | ||
| py.test --junitxml tests/py/junit-test-results.xml bindings/py/tests | ||
| - run: | ||
| name: Running C++ Tests | ||
| command: | | ||
| mkdir -p tests/cpp | ||
| pushd build/release/bin | ||
| ./cpp_region_test | ||
| ./py_region_test | ||
| ./helloregion | ||
| ./hello_sp_tp | ||
| ./prototest | ||
| ./unit_tests --gtest_output=xml:$CIRCLE_TEST_REPORTS/unit_tests_report.xml | ||
| - run: | ||
| name: Running python tests | ||
| command: | | ||
| pip install --user --no-index --find-links=`pwd`/dist/ nupic.bindings | ||
| py.test bindings/py/tests | ||
| ./unit_tests --gtest_output=xml:$CIRCLE_WORKING_DIRECTORY/tests/cpp/unit_tests_report.xml | ||
| # Collect artifacts | ||
| - run: | ||
| name: Collecting artifacts | ||
| command: | | ||
| cp $CIRCLE_TEST_REPORTS/unit_tests_report.xml $CIRCLE_ARTIFACTS/ | ||
| cp dist/*.whl $CIRCLE_ARTIFACTS/ | ||
| - run: | ||
| name: Deploying to S3 | ||
| command: | | ||
| if [[ "$CIRCLE_REPOSITORY_URL" == "https://github.com/numenta/nupic.core" ]]; then | ||
| ci/circle/deploy_s3-osx.sh; | ||
| else | ||
| echo "Skipping deployment, as this is a fork"; | ||
| fi | ||
| - store_test_results: | ||
| path: tests | ||
|
|
||
| - store_artifacts: | ||
| path: dist/*.whl | ||
|
|
||
| - persist_to_workspace: | ||
| root: dist | ||
| paths: | ||
| - nupic.bindings*.whl | ||
| - requirements.txt | ||
| - include/nupic | ||
|
|
||
| deploy-s3: | ||
| machine: true | ||
| steps: | ||
| - attach_workspace: | ||
| at: dist | ||
| - run: | ||
| name: Deploying to S3 | ||
| command: | | ||
| pip install awscli --upgrade --user | ||
| tar -zcv -f nupic_core-${CIRCLE_SHA1}-darwin64.tar.gz dist | ||
| aws s3 cp nupic_core-${CIRCLE_SHA1}-darwin64.tar.gz s3://artifacts.numenta.org/numenta/nupic.core/circle/ | ||
| # Teardown | ||
| # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each | ||
| # The following line was run implicitly in your 1.0 builds based on what CircleCI inferred about the structure of your project. In 2.0 you need to be explicit about which commands should be run. In some cases you can discard inferred commands if they are not relevant to your project. | ||
| - run: | ||
| name: Teardown | ||
| command: find ~/Library/Developer/Xcode/DerivedData -name '*.xcactivitylog' -exec cp {} $CIRCLE_ARTIFACTS/xcactivitylog \; || true | ||
| # Save test results | ||
| - store_test_results: | ||
| path: /tmp/circleci-test-results | ||
| # Save artifacts | ||
| - store_artifacts: | ||
| path: /tmp/circleci-artifacts | ||
| workflows: | ||
| version: 2 | ||
| build-test-deploy: | ||
| jobs: | ||
| - build-and-test | ||
| - deploy-s3: | ||
| requires: | ||
| - build-and-test | ||
| filters: | ||
| branches: | ||
| only: master |
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.