changes to make it easier to get started developing

DEVELOPMENT.md

@@ -1,25 +1,43 @@
 # Development Notes
 
-## Run development tasks
+## Developing
 
-In order to run in the appropriate development environment, you need to run
-the following from the top-level directory with the docker-compose.yml file.
-This bel_operations docker instance includes all of the python modules needed
-for the main bel_api instance as well as additional python development modules
-such as py.test
+After going through the setup. You will be able to edit the bel and belapi code and see
+the changes made to the docker containers immediately. The belapi code is volume mounted
+into the /app directory of the docker container. The bel code is volume mounted into the
+python 3.6 package location for the install bel package.
 
+## Setup
 
-    docker-compose run bel_operations
+To get started:
 
-This will start a docker container with a python environment setup for testing, etc.
+    bash <(curl -s https://raw.githubusercontent.com/belbio/bel_api/master/bin/setup_dev.sh)
 
-You may want to add the following to your .bash_aliases file
+Review configuration in the .env file and the conf directory files
 
-    alias belops="docker-compose run bel_operations"
+Run:
 
-You can now run tests inside the bel_operations instance:
+    docker-compose up -d
+
+## Docker info
+
+    # Follows logs for the belapi service
+    docker-compose logs -f bb_belapi
+
+Services:
+    bb_belapi - provides BEL API endpoint
+    bb_elasticsearch - used for terminology searches/term completion
+    bb_arangodb - [graph] database for terms, orthologies, etc
+    bb_celery_worker - queue/batch processing manager
+    bb_flower - Web GUI for celery
+    bb_rabbitmq - Message broker used for queues by celery
+    bb_traefik - reverse proxy to provide access to the docker services
+
+## Misc
+
+This will start docker containers needed to run everything.
+
+To make life easier on Macs using local DNS dev domains: https://medium.com/@williamhayes/local-dev-on-docker-fun-with-dns-85ca7d701f0a
 
-    ./bin/runtests
 
-The bel_operations prompt will look like `api@<container_id>:/app$`
 

bin/setup_dev.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+URLROOT=https://raw.githubusercontent.com/belbio/bel_api/master
+
+# Create directories
+mkdir conf
+mkdir bel_specifications
+
+# Clone repos
+git clone git@github.com:belbio/bel_api.git belapi
+git clone git@github.com:belbio/bel.git
+
+# Set up configuration
+curl ${URLROOT}/conf/dotenv.sample -o .env
+curl ${URLROOT}/conf/belbio_conf.yml.sample -o conf/belbio_conf.yml
+curl ${URLROOT}/conf/elasticsearch.yml -o conf/elasticsearch.yml
+touch conf/belbio_secrets.yml
+
+curl ${URLROOT}/docker-compose.dev.yml -o docker-compose.yml
+
+docker network create belbio
+
+echo "Run following command to start belapi"
+echo "  docker-compose up -d"
+

conf/belbio_conf.yml.sample

@@ -10,7 +10,7 @@
 # bel package settings
 bel:
   lang:
-    default_bel_version: 2.0.0
+    default_bel_version: 2.1.0
 
     # Location of BEL Language Specification yaml files for different versions
     specifications: /belbio/bel_specifications
@@ -63,12 +63,12 @@ bel_api:
   servers:
     server_type: DEV
 
-    api_url: http://belapi.dev.biodati.test
+    api_url: http://belapi.dev.belbio.test
 
-    elasticsearch: http://elasticsearch:9200
+    elasticsearch: http://bb_elasticsearch:9200
 
     # arangodb_host: docker.for.mac.localhost
-    arangodb_host: arangodb
+    arangodb_host: bb_arangodb
     arangodb_protocol: http
     arangodb_port: 8529
     arangodb_username: ''

conf/belbio_secrets.yml.sample

@@ -0,0 +1,14 @@
+# belbio secrets - added to config['secrets']
+
+# Location of this file can be found in one of the following locations
+#   first found is the one that is used
+#   - ./belbio_secrets.yml    (just a regular file in the local directory or any parent directory)
+#   - ~/.belbio_secrets       (a dotfile in the home directory)
+
+
+bel_api:
+  shared_secret: xxx  # shared secret to validate JWT token
+  servers:
+      arangodb_password: ''
+  mail:
+    api_key: xxx

conf/dotenv.sample

@@ -1,6 +1,10 @@
-BS_SERVER_MODE=DEV
+SERVICES_HOST_NAME=dev.belbio.test
+SERVER_MODE=DEV
 BELBIO_CONF=/belbio
 # GITHUB_ACCESS_TOKEN=
 ARANGO_USER=root
 ARANGO_PASSWD=root
 JWT_SECRET=testing
+RABBITMQ_USER=admin
+RABBITMQ_PASSWD=admin
+

docker-compose.dev.yml

@@ -9,33 +9,32 @@ volumes:
     driver: local
 
 networks:
-  biodati:
+  belbio:
     external: true
 
 services:
+
   # BEL API - core requirement
   # CELERY DOES NOT SUPPORT Python 3.7 yet - due to async keyword!!!!!!!!!!
-  belapi:
-    container_name: belapi
+  bb_belapi:
+    container_name: bb_belapi
     image: belbio/bel_api:localdev
     build:
       context: ./belapi
       dockerfile: ./docker/Dockerfile-bel_api-dev
-    ports:
-      - "8000:8000"
     environment:
-      - CELERY_BROKER=amqp://${BS_RABBITMQ_USER:?err}:${BS_RABBITMQ_PASSWD:?err}@rabbitmq:5672
-      - CELERY_BACKEND=rpc://${BS_RABBITMQ_USER:?err}:${BS_RABBITMQ_PASSWD:?err}@rabbitmq:5672
-      - SERVER_MODE=${BS_SERVER_MODE:?err}
+      - CELERY_BROKER=amqp://${RABBITMQ_USER:?err}:${RABBITMQ_PASSWD:?err}@rabbitmq:5672
+      - CELERY_BACKEND=rpc://${RABBITMQ_USER:?err}:${RABBITMQ_PASSWD:?err}@rabbitmq:5672
+      - SERVER_MODE=${SERVER_MODE:?err}
       - BELBIO_CONF=${BELBIO_CONF:?err}
       - GITHUB_ACCESS_TOKEN=${GITHUB_ACCESS_TOKEN}
-      - ARANGO_USER=${BS_ARANGO_USER}
-      - ARANGO_PASSWD=${BS_ARANGO_PASSWD}
-      - JWT_SECRET=${BS_Auth0_ClientSecret:?err}
+      - ARANGO_USER=${ARANGO_USER}
+      - ARANGO_PASSWD=${ARANGO_PASSWD}
+      - JWT_SECRET=${JWT_SECRET:?err}
     depends_on:
-      - rabbitmq
-      - elasticsearch
-      - arangodb
+      - bb_rabbitmq
+      - bb_elasticsearch
+      - bb_arangodb
     volumes:
       - ./belapi/api:/app
       # Add bel python package so it's editable
@@ -45,11 +44,11 @@ services:
       - ./conf/belbio_conf.yml:/belbio/belbio_conf.yml
       - ./conf/belbio_secrets.yml:/belbio/belbio_secrets.yml
     networks:
-      biodati:
+      belbio:
     labels:
       - traefik.enable=true
       - traefik.backend=belapi
-      - traefik.frontend.rule=Host:belapi.${BS_HOST_NAME:?err};
+      - traefik.frontend.rule=Host:belapi.${SERVICES_HOST_NAME:?err};
       - traefik.port=8000
       - traefik.docker.network=biodati
     logging:
@@ -63,8 +62,8 @@ services:
   # ArangoDB - document/graph store - core requirement
   # http://localhost:8529
   # Upgrading - run this first:  "docker-compose run --rm arangodb arangod --database.auto-upgrade"
-  arangodb:
-    container_name: arangodb
+  bb_arangodb:
+    container_name: bb_arangodb
     image: arangodb:3.4
     command: --log.level info --log.level performance=info --log.level queries=info --log.level requests=warning
     ports:
@@ -75,7 +74,7 @@ services:
       - ARANGO_NO_AUTH=1
       # - ARANGO_ROOT_PASSWORD=${BS_ARANGO_PASSWD:?err}
     networks:
-      biodati:
+      belbio:
     logging:
       driver: "json-file"
       options:
@@ -86,16 +85,16 @@ services:
     restart: unless-stopped
 
   # Elasticsearch 6.0
-  elasticsearch:
-    container_name: elasticsearch
+  bb_elasticsearch:
+    container_name: bb_elasticsearch
     image: docker.elastic.co/elasticsearch/elasticsearch:6.5.4
     ports:
       - "9200:9200"
     volumes:
       - dev_elasticsearch_data:/usr/share/elasticsearch/data
       - ./conf/elasticsearch.yml:/conf/elasticsearch.yml
     networks:
-      biodati:
+      belbio:
     environment:
       - cluster.name=docker-cluster
       - bootstrap.memory_lock=true
@@ -118,7 +117,7 @@ services:
         max-file: "3"
     restart: unless-stopped
 
-  celery_worker:
+  bb_celery_worker:
     image: belbio/bel_api:localdev
     build:
       context: .
@@ -133,33 +132,33 @@ services:
       - ./conf/belbio_secrets.yml:/belbio/belbio_secrets.yml
     command: celery -A services.tasks:celery_app worker --loglevel=info
     networks:
-      biodati:
+      belbio:
     environment:
-      - SERVER_MODE=${BS_SERVER_MODE:?err}
-      - CELERY_BROKER=amqp://${BS_RABBITMQ_USER:?err}:${BS_RABBITMQ_PASSWD:?err}@rabbitmq:5672
-      - CELERY_BACKEND=rpc://${BS_RABBITMQ_USER:?err}:${BS_RABBITMQ_PASSWD:?err}@rabbitmq:5672
+      - SERVER_MODE=${SERVER_MODE:?err}
+      - CELERY_BROKER=amqp://${RABBITMQ_USER:?err}:${RABBITMQ_PASSWD:?err}@rabbitmq:5672
+      - CELERY_BACKEND=rpc://${RABBITMQ_USER:?err}:${RABBITMQ_PASSWD:?err}@rabbitmq:5672
       - BELBIO_CONF=${BELBIO_CONF:?err}
       - GITHUB_ACCESS_TOKEN=${GITHUB_ACCESS_TOKEN}
     depends_on:
-      - belapi
-      - rabbitmq
+      - bb_belapi
+      - bb_rabbitmq
     logging:
       driver: "json-file"
       options:
         max-size: "5k"
         max-file: "3"
     restart: unless-stopped
 
-  flower:
-    container_name: flower
+  bb_flower:
+    container_name: bb_flower
     image: belbio/bel_api:localdev
     build:
       context: .
       dockerfile: ./belapi/docker/Dockerfile-bel_api-dev
     environment:
-      - SERVER_MODE=${BS_SERVER_MODE:?err}
-      - CELERY_BROKER=amqp://${BS_RABBITMQ_USER:?err}:${BS_RABBITMQ_PASSWD:?err}@rabbitmq:5672
-      - CELERY_BACKEND=rpc://${BS_RABBITMQ_USER:?err}:${BS_RABBITMQ_PASSWD:?err}@rabbitmq:5672
+      - SERVER_MODE=${SERVER_MODE:?err}
+      - CELERY_BROKER=amqp://${RABBITMQ_USER:?err}:${RABBITMQ_PASSWD:?err}@rabbitmq:5672
+      - CELERY_BACKEND=rpc://${RABBITMQ_USER:?err}:${RABBITMQ_PASSWD:?err}@rabbitmq:5672
       - BELBIO_CONF=${BELBIO_CONF:?err}
       - GITHUB_ACCESS_TOKEN=${GITHUB_ACCESS_TOKEN}
     ports:
@@ -172,18 +171,18 @@ services:
       - ./bel_specifications:/belbio/bel_specifications
       - ./conf/belbio_conf.yml:/belbio/belbio_conf.yml
       - ./conf/belbio_secrets.yml:/belbio/belbio_secrets.yml
-    command:  flower -A services.tasks:celery_app --port=5555 --broker=amqp://${BS_RABBITMQ_USER:?err}:${BS_RABBITMQ_PASSWD:?err}@rabbitmq:5672
+    command:  flower -A services.tasks:celery_app --port=5555 --broker=amqp://${RABBITMQ_USER:?err}:${RABBITMQ_PASSWD:?err}@rabbitmq:5672
     networks:
-      biodati:
+      belbio:
     labels:
       - traefik.enable=true
       - traefik.backend=flower
-      - traefik.frontend.rule=Host:flower.${BS_HOST_NAME:?err}
+      - traefik.frontend.rule=Host:flower.${SERVICES_HOST_NAME:?err}
       - traefik.port=5555
       - traefik.docker.network=biodati
     depends_on:
-      - belapi
-      - rabbitmq
+      - bb_belapi
+      - bb_rabbitmq
     logging:
       driver: "json-file"
       options:
@@ -192,26 +191,23 @@ services:
     restart: unless-stopped
 
   # Used by celery for a message queue
-  rabbitmq:
-    container_name: rabbitmq
+  bb_rabbitmq:
+    container_name: bb_rabbitmq
     hostname: rabbitmq
     image: rabbitmq:3.6-management
     environment:
-      - RABBITMQ_DEFAULT_USER=${BS_RABBITMQ_USER:?err}
-      - RABBITMQ_DEFAULT_PASS=${BS_RABBITMQ_PASSWD:?err}
+      - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:?err}
+      - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWD:?err}
       - RABBITMQ_VM_MEMORY_HIGH_WATERMARK=.2  # Allow up to 20% of RAM
       - RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbit log_levels [{connection,error}]
     volumes:
       - dev_rabbitmq_data:/var/lib/rabbitmq
-    ports:
-      - 5672:5672
-      - 15672:15672
     networks:
-      biodati:
+      belbio:
     labels:
       - traefik.enable=true
       - traefik.backend=rabbitmq
-      - traefik.frontend.rule=Host:rabbitmq.${BS_HOST_NAME:?err}
+      - traefik.frontend.rule=Host:rabbitmq.${SERVICES_HOST_NAME:?err}
       - traefik.port=15672
       - traefik.docker.network=biodati
     logging:
@@ -220,3 +216,29 @@ services:
         max-size: "5k"
         max-file: "3"
     restart: unless-stopped
+
+  bb_traefik:
+    container_name: bb_traefik
+    image: traefik:1.7.4
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./conf/traefik.toml:/traefik.toml
+      - ./logs:/logs
+    networks:
+      belbio:
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10k"
+        max-file: "3"
+    # network_mode: "host"
+    labels:
+      - traefik.enable=true
+      - traefik.backend=traefik
+      - traefik.frontend.rule=Host:traefik.${SERVICES_HOST_NAME:?err}
+      - traefik.port=8080
+      - traefik.docker.network=biodati
+    restart: "unless-stopped"