Merge pull request #119 from goanpeca/fix/setup

PR: Simplify setup to make it more standard and create scripts in scripts/ folder

.gitignore

@@ -97,8 +97,6 @@ htmlcov
 /.anaconda
 /.envs
 /record.txt
-# version.py is a generated file
-/anaconda_project/version.py
 /docs/build
 /examples/movies/movies.db
 *.orig

.travis.yml

@@ -1,3 +1,5 @@
+# https://travis-ci.org/Anaconda-Platform/anaconda-project/
+
 # language: python sets up virtualenv and pip that we don't need.
 # omitting language gives us ruby stuff. c seems likely to be a minimal setup.
 language: c
@@ -78,10 +80,10 @@ install:
     fi ;
 
 script:
-  - if test "$TEST_TARGET" = tests; then LANG=en_US.UTF-8 python setup.py test -a '-vv'; fi
+  - if test "$TEST_TARGET" = tests; then LANG=en_US.UTF-8 python scripts/run_tests.py --pytest-args ' -vv'; fi
   - if test "$TRAVIS_PYTHON_VERSION" = "3.5" && test "$TEST_TARGET" = "packaging"; then
       git fetch --unshallow ;
-      LANG=en_US.UTF-8 python setup.py conda_package ;
+      LANG=en_US.UTF-8 python scripts/create_conda_packages.py;
     else
       echo "Not building packages on $TRAVIS_PYTHON_VERSION target=$TEST_TARGET" ;
     fi

README.md

@@ -1,6 +1,18 @@
 # Anaconda Project
 
-*Reproducible, executable project directories.*
+*Create, load and manipulate reproducible data science projects.*
+
+## Build status
+
+[![Build Status](https://travis-ci.org/Anaconda-Platform/anaconda-project.svg?branch=rename-to-anaconda-project)](https://travis-ci.org/Anaconda-Platform/anaconda-project)
+[![Build status](https://ci.appveyor.com/api/projects/status/54dmsnyeuutfvutj?svg=true)](https://ci.appveyor.com/project/ContinuumAnalytics/anaconda-project-fqvvy)
+
+## Project information
+
+[![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
+
+
+## Description
 
 Take any directory full of stuff that you're working on; web apps,
 scripts, Jupyter notebooks, data files, whatever it may be.

RELEASE.md

@@ -0,0 +1,72 @@
+# Release process
+
+## To release a new version of **anaconda-project** on PyPI:
+
+**1.)** Ensure you have the latest version from upstream and update your fork
+
+    git pull upstream master
+    git push origin master
+
+**2.)** Update [CHANGELOG.md](https://github.com/anaconda-platform/anaconda-project/blob/master/CHANGELOG.md), using loghub itself
+
+    loghub anaconda-platform/anaconda-project -m <milestone>
+
+**3.)** Update [`anaconda_project/version.py`](https://github.com/anaconda-platform/anaconda-project/blob/master/anaconda_project/version.py) (set release version, remove 'dev0')
+
+**4.)** Commit changes
+
+    git add .
+    git commit -m "Set release version"
+
+**5.)** Create distributions
+
+    python setup.py sdist bdist_wheel
+
+**6.)** Upload distributions
+
+    twine upload dist/* -u <username> -p <password>
+
+**7.)** Add release tag
+
+    git tag -a vX.X.X -m 'Tag release version vX.X.X'
+
+**8.)** Update `version.py` (add 'dev0' and increment minor or major as needed)
+
+**9.)** Commint changes
+
+    git add . 
+    git commit -m "Restore dev version"
+
+**10.)** Push changes
+
+    git push upstream master
+    git push origin master
+    git push --tags
+
+
+## To release a new version of **anaconda-project** on anaconda-platform:
+
+**1.)** After the previous steps on pypi (and tagging) run:
+
+    bash scripts/build_and_upload.sh
+
+## To release a new version of **anaconda-project** on conda-forge:
+
+**1.)** Ensure you have the latest version from upstream and update your fork
+
+    git pull upstream master
+    git push origin master
+
+**2.)** Update [meta.yaml](https://github.com/anaconda-platform/anaaconda-project/blob/master/conda.recipe/meta.yaml) version and hash
+
+**3.)** Commit changes
+
+    git add .
+    git commit -m "Update conda recipe"
+
+**4.)** Update recipe on [conda-forge/anaconda-project-feedstock](https://github.com/conda-forge/anaconda-project-feedstock)
+
+**5.)** Push changes
+
+    git push upstream master
+    git push origin master

anaconda_project/version.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------------
+# Copyright (c) 2016, Anaconda, Inc. All rights reserved.
+#
+# Licensed under the terms of the BSD 3-Clause License.
+# The full license is in the file LICENSE.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+"""Version of anaconda-project library."""
+
+VERSION_INFO = (0, 8, 0, 'dev0')
+__version__ = '.'.join(map(str, VERSION_INFO))
+version = __version__

appveyor.yml

@@ -1,12 +1,15 @@
-environment:
+# https://ci.appveyor.com/project/ContinuumAnalytics/anaconda-project-fqvvy/
+
+matrix:
+  fast_finish: true
 
+environment:
   global:
     PYTHON: "C:\\conda"
     CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci-helpers\\appveyor\\windows_sdk.cmd"
     PYTHON_ARCH: "64" # needs to be set for CMD_IN_ENV to succeed. If a mix
                       # of 32 bit and 64 bit builds are needed, move this
                       # to the matrix section.
-
   matrix:
     - PYTHON_VERSION: "2.7"
     - PYTHON_VERSION: "3.5"
@@ -52,4 +55,4 @@ test_script:
   - python -c "import sys; print(sys.executable)"
   - python -c "import sys; print(sys.prefix)"
   - python --version
-  - python setup.py test
+  - python scripts/run_tests.py

conda.recipe/meta.yaml

@@ -4,12 +4,14 @@ package:
 
 source:
   git_url: ../
+  git_tag: {% if environ.get('PRERELEASE', '') %} HEAD {% else %} {{ environ.get('GIT_DESCRIBE_TAG') }} {% endif %}
   # If you're trying to test build changes, you have to add this
   # pointing to the branch you're modifying.
   # git_tag: package-script
 
 build:
   number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }}
+  script: python setup.py install --single-version-externally-managed --record=record.txt
   entry_points:
     - anaconda-project = anaconda_project.cli:main
 

build_and_upload.sh → scripts/build_and_upload.sh

@@ -1,13 +1,12 @@
 #!/bin/bash
-
 set -e
 
 # be sure we are in the right place
 test -e setup.py || exit 1
 test -d anaconda_project || exit 1
 
 (test -d build/packages && /bin/rm -r build/packages) || true
-python setup.py test
-python setup.py conda_package
+python scripts/run_tests.py
+python scripts/create_conda_packages
 
 anaconda upload -u anaconda-platform --label dev build/packages/**/**/*.tar.bz2

scripts/create_conda_packages.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+# -----------------------------------------------------------------------------
+# Copyright (c) 2016, Anaconda, Inc. All rights reserved.
+#
+# Licensed under the terms of the BSD 3-Clause License.
+# The full license is in the file LICENSE.txt, distributed with this software.
+# -----------------------------------------------------------------------------
+"""Script to automate the creation of conda packages for a release."""
+
+from __future__ import print_function
+
+# Standard library imports
+import argparse
+import ast
+import os
+import subprocess
+import shutil
+import sys
+
+# Constants
+HERE = os.path.abspath(os.path.dirname(__file__))
+ROOT = os.path.dirname(HERE)
+PYTHON_VERSIONS = ('2.7', '3.5', '3.6')
+
+
+def get_version(module='anaconda_project'):
+    """Get version."""
+    with open(os.path.join(ROOT, module, 'version.py'), 'r') as f:
+        data = f.read()
+    lines = data.split('\n')
+    for line in lines:
+        if line.startswith('VERSION_INFO'):
+            version_tuple = ast.literal_eval(line.split('=')[-1].strip())
+            version = '.'.join(map(str, version_tuple))
+            break
+    return version
+
+
+VERSION = get_version()
+
+
+class CondaPackageCreator:
+    "Create Conda packages"
+
+    def __init__(self, packages_dir=None):
+        "Create Conda packages"
+        self.packages_dir = packages_dir
+        self.clean_up()
+
+        if self.packages_dir is None:
+            self.packages_dir = os.path.join(ROOT, 'build', 'packages')
+        self._safe_makedirs(self.packages_dir)
+
+    def clean_up(self):
+        # Clean up leftover trash as best we can
+        BUILD_TMP = os.path.join(ROOT, 'build', 'tmp')
+        if os.path.isdir(BUILD_TMP):
+            print("Cleaning up " + BUILD_TMP)
+            try:
+                shutil.rmtree(BUILD_TMP, ignore_errors=True)
+            except Exception as e:
+                print("Failed to remove %s: %s" % (BUILD_TMP, str(e)))
+            else:
+                print("Done removing " + BUILD_TMP)
+
+    def run(self):
+        try:
+            self._real_run()
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            print("setup.py: Failed to build packages: " + str(e), file=sys.stderr)
+            sys.exit(1)
+
+    def _real_run(self):
+        recipe_dir = os.path.join(ROOT, 'conda.recipe')
+        python_versions = PYTHON_VERSIONS
+        all_final_package_paths = []
+        for python_version in python_versions:
+            out = subprocess.check_output(['conda-build', '--output', '--python', python_version, recipe_dir])
+            package_path = out.decode('utf-8').strip()
+            print("expected conda package path: " + package_path)
+            if '--' in package_path:
+                # conda build bug?
+                print("package_path looks broken, contains -- in it. fixing...")
+                package_path = package_path.replace("--", "-%s-" % VERSION)
+                print("new conda package path: " + package_path)
+            build_arch = os.path.basename(os.path.dirname(package_path))
+            python_scoped_package_dir = os.path.join(self.packages_dir, "py%s" % python_version)
+            final_package_path = os.path.join(python_scoped_package_dir, build_arch, os.path.basename(package_path))
+            all_final_package_paths.append(final_package_path)
+            if os.path.isfile(final_package_path):
+                print("Package for python %s platform %s already exists: %s" %
+                      (python_version, build_arch, final_package_path))
+            else:
+                if os.path.isfile(package_path):
+                    print("Already built for python %s at %s" % (python_version, package_path))
+                else:
+                    print("Calling conda build for %s %s" % (python_version, build_arch))
+                    code = subprocess.call(['conda', 'build', '--no-binstar-upload', '--python', python_version,
+                                            recipe_dir])
+                    if code != 0:
+                        raise Exception("Failed to build for python version " + python_version)
+                    if not os.path.isfile(package_path):
+                        try:
+                            print("files that DO exist: " + repr(os.listdir(os.path.basename(package_path))))
+                        except Exception as e:
+                            print(" (failed to list files that do exist, %s)" % str(e))
+                        raise Exception("conda said it would build %s but it didn't" % package_path)
+
+                self._safe_makedirs(os.path.dirname(final_package_path))
+                print("Copying %s to %s" % (package_path, final_package_path))
+                shutil.copyfile(package_path, final_package_path)
+                print("Created %s" % final_package_path)
+
+            for arch in ('osx-64', 'linux-32', 'win-32', 'win-64'):
+                if arch == build_arch:
+                    continue
+                converted_output_dir = os.path.join(python_scoped_package_dir)
+                converted_package_path = os.path.join(converted_output_dir, arch, os.path.basename(package_path))
+                if os.path.isfile(converted_package_path):
+                    print("Already converted to %s from %s for python %s" % (arch, build_arch, python_version))
+                else:
+                    print("Creating %s by conversion %s=>%s" % (converted_package_path, build_arch, arch))
+                    self._safe_makedirs(converted_output_dir)
+                    # this automatically creates the "arch" directory to put the package in
+                    code = subprocess.call(['conda', 'convert', '--platform', arch, final_package_path, '--output-dir',
+                                            converted_output_dir])
+                    if code != 0:
+                        raise Exception("Failed to convert from %s to %s to create %s" %
+                                        (build_arch, arch, converted_package_path))
+                    all_final_package_paths.append(converted_package_path)
+
+        print("Packages in " + self.packages_dir)
+
+    def _safe_makedirs(self, path):
+        try:
+            os.makedirs(path)
+        except OSError:
+            pass
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Script to create conda packages for all platforms and '
+                                     'python version 2.7, 3.5 and 3.6')
+    parser.add_argument(action="store",
+                        dest="packages_dir",
+                        default=None,
+                        help="location directory for built packages",
+                        nargs='?')
+
+    options = parser.parse_args()
+    cpc = CondaPackageCreator(packages_dir=options.packages_dir)
+    cpc.run()
+
+
+if __name__ == '__main__':
+    main()