Update deployment process to use conda/pip instead of cloning github repos

.travis.yml

@@ -14,11 +14,12 @@ install:
   - popd
   - source activate aei_dropq
 
+
 addons:
     postgresql: "9.4"
 
 before_script:
   - psql -c 'create database test_db;' -U postgres
-  
+
 script:
   - py.test webapp/apps/

RELEASES.md

@@ -4,6 +4,17 @@ Go
 [here](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pulls?q=is%3Apr+is%3Aclosed)
 for a complete commit history.
 
+Release 1.4.1 on 2018-02-27
+----------------------------
+**Major Changes**
+- None
+
+**Minor Changes**
+- None
+
+**Bug Fixes**
+- [#826](https://github.com/OpenSourcePolicyCenter/PolicyBrain/pull/826) - Update deployment process to use conda/pip instead of cloning github repos - Hank Doupe
+
 Release 1.4.0 on 2018-02-26
 ----------------------------
 **Major Changes**

conda-requirements.txt

@@ -1,9 +1,10 @@
+gevent
+pillow
+pyparsing
+bokeh=0.12.7
 nomkl
 taxcalc==0.15.2
 btax==0.2.2
+ogusa==0.5.8
 numba>=0.33.0
 pandas>=0.22.0
-gevent
-pillow
-pyparsing
-bokeh=0.12.7

deploy/fab/dropq_environment.yml

@@ -1,13 +1,23 @@
 name: aei_dropq
 dependencies:
 - python<=2.7.14,>=2.7.12
-- pandas>=0.22.0
-- flask
-- greenlet
+- numpy >=1.12.1
+- pandas >=0.22.0
+- taxcalc =0.15.2
+- btax =0.2.2
+- ogusa =0.5.8
+- numba
+- six
+- bokeh =0.12.7
+- mock
 - xlrd
-- redis
 - sphinx
+- nomkl
 - pip:
+  - flask
+  - redis
+  - greenlet
+  - toolz
   - fabric
   - boto
   - celery
@@ -16,8 +26,16 @@ dependencies:
   - plotly
   - sphinx-rtd-theme
   - sphinx-autobuild
+  - redis
+  - supervisor
+  - plotly
+  - requests-mock
+  - requests
+  - funcsigs
+  - xlrd >=0.9.0
 
 channels:
   - ospc
+  - ospc/label/dev
   - defaults
   - conda-forge

deploy/fab/redeploy.py

@@ -28,19 +28,7 @@ def cli(ip_address=None):
     parser = argparse.ArgumentParser(description='Re-deploy the deploy app from laptop to EC2 box that already has deploy on it')
     if not ip_address:
         parser.add_argument('ip_address', help='IP address that already has been deployed (has reset_server.sh in home dir)')
-    parser.add_argument('taxcalc_version', help="Tax-Calculator git tag or conda version")
-    parser.add_argument('taxcalc_install_method', choices=('git', 'conda'), help="Install method - choices: %(choices)s")
-    parser.add_argument('btax_version', help="B-Tax git tag or conda version")
-    parser.add_argument('btax_install_method', choices=('git', 'conda'), help="Install method - choices: %(choices)s")
-    parser.add_argument('ogusa_version', help="OG-USA git tag or conda version")
-    parser.add_argument('ogusa_install_method', choices=('git', 'conda'), help="Install method - choices: %(choices)s")
     parser.add_argument('--keypath', default="", help='Full local path to the PEM file for EC2 box access, typically something like /path/to/latest.pem')
-    parser.add_argument('--taxcalc_install_label', default=' -c ospc ',
-                        help="Conda label for ospc's taxcalc: Default- %(default)s")
-    parser.add_argument('--ogusa_install_label', default=' -c ospc ',
-                        help="Conda label for ospc's ogusa: Default- %(default)s")
-    parser.add_argument('--btax_install_label', default=' -c ospc ',
-                        help="Conda label for ospc's btax: Default- %(default)s")
     parser.add_argument('--user', default='ubuntu', help='Login user, typically (default): %(default)s')
     parser.add_argument('--allow-uncommited',
                         action='store_true',
@@ -81,11 +69,6 @@ def run(pem, ip, fname, cmd):
 
 def main(args=None):
     args = args or cli()
-    env_str = []
-    for k in dir(args):
-        if 'install_method' in k or 'version' in k or 'install_label' in k:
-            os.environ[k.upper()] = getattr(args, k)
-            env_str.append('{}="{}"'.format(k.upper(), getattr(args, k)))
     fname = next_log_file(args)
     if args.keypath:
         assert os.path.exists(args.keypath), ('PEM file {} does not exist'.format(args.keypath))
@@ -101,8 +84,8 @@ def main(args=None):
     put_func = lambda x, y: put(pem, ip_address, fname, x, y)
     run_func = lambda cmd: run(pem, ip_address, fname, cmd)
     copy_deploy_repo(None, put_func, run_func)
-    template = 'ssh {} {}@{} \'{} bash reset_server.sh\''
-    cmd = template.format(pem, user, ip_address, " ".join(env_str))
+    template = 'ssh {} {}@{} \'bash reset_server.sh\''
+    cmd = template.format(pem, user, ip_address)
     proc_mgr(cmd, fname)
 
 

deploy/fab/reset_server.sh

@@ -2,13 +2,17 @@
 export rs="reset_server.sh STATUS: "
 echo $rs activate aei_dropq
 export DEP=/home/ubuntu/deploy
+export PATH="/home/ubuntu/miniconda2/bin:$PATH"
+
+conda config --set always_yes yes --set changeps1 no
+conda clean --all
+conda env remove --name aei_dropq
+conda env create -f $DEP/fab/dropq_environment.yml
+
 source /home/ubuntu/miniconda2/bin/activate aei_dropq
 pushd ${DEP}
 python setup.py install
 popd
-conda config --set always_yes true
-conda clean --all
-conda install pandas=0.20.1
 echo $rs get taxpuf package
 cd $DEP
 export TAXPUF_CHANNEL="https://conda.anaconda.org/t/$(cat /home/ubuntu/.ospc_anaconda_token)/opensourcepolicycenter"
@@ -28,66 +32,22 @@ for repeat in 1 2 3;
     done
 cd $DEP/..
 echo $rs configure conda
-conda config --set always_yes yes --set changeps1 no
 echo $rs remove old versions
-conda remove taxcalc; pip uninstall -y taxcalc
-conda remove ogusa ; pip uninstall -y ogusa
-conda remove btax ; pip uninstall -y btax
 echo $rs Install taxcalc
 cd $DEP/.. && rm -rf Tax-Calculator B-Tax OG-USA
-git clone http://github.com/open-source-economics/Tax-Calculator
-cd Tax-Calculator && git fetch --all && git fetch origin --tags && git checkout $TAXCALC_VERSION
-if [ "$TAXCALC_INSTALL_LABEL" = "" ];then
-    export TAXCALC_INSTALL_LABEL=" -c ospc ";
-fi
-if [ "$BTAX_INSTALL_LABEL" = "" ];then
-    export BTAX_INSTALL_LABEL=" -c ospc ";
-fi
-if [ "$OGUSA_INSTALL_LABEL" = "" ];then
-    export OGUSA_INSTALL_LABEL=" -c ospc ";
-fi
-if [ "$TAXCALC_INSTALL_METHOD" = "git" ];then
-    python setup.py install
-else
-    conda install $TAXCALC_INSTALL_LABEL taxcalc=$TAXCALC_VERSION
-    conda list | grep 'taxcalc' | awk -F' ' '{print $2}' | xargs -n 1 git checkout
-fi
-echo $rs Install OG-USA
-if [ "$OGUSA_INSTALL_METHOD" = "git" ];then
-    cd $DEP/..
-    git clone http://github.com/open-source-economics/OG-USA
-    cd OG-USA && git fetch --all && git fetch origin --tags && git checkout $OGUSA_VERSION && python setup.py install
-else
-    conda install $OGUSA_INSTALL_LABEL ogusa=$OGUSA_VERSION
-fi
-echo $rs Install B-Tax
-if [ "$BTAX_INSTALL_METHOD" = "git" ];then
-    cd $DEP/..
-    git clone http://github.com/open-source-economics/B-Tax
-    export BTAX_CUR_DIR=`pwd`/B-Tax/btax
-    cd B-Tax && git fetch --all && git fetch origin --tags && git checkout $BTAX_VERSION && python setup.py install
-else
-    conda install $BTAX_INSTALL_LABEL btax=$BTAX_VERSION
-fi
+
+cd $DEP
 
 # TODO LATER
 # conda install -c ospc btax --no-deps
 echo $rs redis-cli FLUSHALL
 redis-cli FLUSHALL
 
-cd ${DEP}/../Tax-Calculator && git fetch origin
-conda list | grep 'taxcalc' | awk -F' ' '{print $2}' | xargs -n 1 git checkout
-cp ~/deploy/puf.csv.gz ./ && gunzip -k puf.csv.gz
-cd $DEP/taxbrain_server/tests
-echo $rs Test the mock celery - mock flask tests in deploy
-MOCK_CELERY=1 TAX_ESTIMATE_PATH=$OGUSA_PATH py.test -p no:django -v
-cd $DEP
 conda list
 echo $rs Remove asset_data.pkl and recreate it with btax execute
 rm -f asset_data.pkl
 python -c "from btax.execute import runner;runner(False,2013,{})"
 echo $rs supervisorctl -c $SUPERVISORD_CONF start all
-conda clean --all
 supervisorctl -c $SUPERVISORD_CONF start all
 python -c "from taxcalc import *;from btax import *;from ogusa import *" && ps ax | grep flask | grep python && ps ax | grep celery | grep python && echo $rs RUNNING FLASK AND CELERY PIDS ABOVE
 echo $rs DONE - OK

deploy/install_taxbrain_server.sh

@@ -11,8 +11,6 @@ install_conda_reqs(){
     for pkg in $(cat ../conda-requirements.txt);do
         echo $pkg | grep -Eoi "(btax)|(ogusa)|(taxcalc)" &> /dev/null || echo install $channel $pkg && conda install $channel $pkg -y || return 1;
     done
-    echo install $channel ogusa -y
-    conda install $channel ogusa -y
 }
 install_reqs(){
     install_conda_reqs || return 1;

deploy/requirements.txt

@@ -1,8 +0,0 @@
-celery
-retrying
-eventlet
-redis
-supervisor
-plotly
-requests-mock
-requests

webapp/settings.py

@@ -49,7 +49,7 @@
 ]
 
 
-WEBAPP_VERSION = "1.4.0"
+WEBAPP_VERSION = "1.4.1"
 
 # Application definition