Merge branch 'tarhan-dev' into dev

.travis.yml

@@ -15,6 +15,11 @@ env:
     init: /sbin/init
     run_opts: ""
   - docker: 1
+    dockerfile_name: Dockerfile
+    docker_tag_suffix: dev
+  - docker: 1
+    dockerfile_name: Dockerfile.py2alpine
+    docker_tag_suffix: dev-alpine
 
 install: true
 

crawler/requirements.txt

@@ -0,0 +1,39 @@
+attrs==16.1.0
+cffi==1.7.0
+ConcurrentLogHandler==0.9.1
+cryptography==1.5
+cssselect==0.9.2
+enum34==1.1.6
+funcsigs==1.0.2
+future==0.15.2
+idna==2.1
+ipaddress==1.0.16
+kafka-python==1.3.1
+kazoo==2.2.1
+lxml==3.6.4
+mock==2.0.0
+nose==1.3.7
+parsel==1.0.3
+pbr==1.10.0
+pyasn1==0.1.9
+pyasn1-modules==0.0.8
+pycparser==2.14
+PyDispatcher==2.0.5
+pyOpenSSL==16.1.0
+python-json-logger==0.1.5
+PyYAML==3.12
+queuelib==1.4.2
+redis==2.10.5
+requests==2.11.1
+requests-file==1.4
+retrying==1.3.3
+Scrapy==1.1.2
+scutils==1.1.0
+service-identity==16.0.0
+six==1.10.0
+testfixtures==4.10.0
+tldextract==2.0.1
+Twisted==16.4.0
+ujson==1.35
+w3lib==1.15.0
+zope.interface==4.2.0

docker/crawler/Dockerfile

@@ -15,7 +15,7 @@ RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # install requirements
-COPY requirements.txt /usr/src/app/
+COPY crawler/requirements.txt /usr/src/app/
 RUN pip install --no-cache-dir -r requirements.txt
 
 # move codebase over

docker/crawler/Dockerfile.py2alpine

@@ -0,0 +1,59 @@
+FROM python:2.7.12-alpine
+MAINTAINER Madison Bahmer <madison.bahmer@istresearch.com>
+
+
+# copy crawler own requirements.txt with its dependencies
+COPY crawler/requirements.txt /usr/src/app/
+# scutils dev version install (not in pip yet)
+# due utils folder located above Dockerfile folder it can not be used
+# for Docker Hub Automated Build feature (but should, see wiki notes)
+COPY utils /tmp/utils
+
+# Combine run command to create single intermeiate image layer
+# This MANDATORY because developments dependencies are huge.
+RUN mkdir -p /usr/src/app \
+ && cd /usr/src/app \
+# Installing runtime dependencies
+ && apk --no-cache add \
+      curl \
+      openssl \
+      libffi \
+      libxml2 \
+      libxslt \
+# Installing buildtime dependencies. They will be removed at end of this
+# commands sequence.
+ && apk --no-cache add --virtual build-dependencies \
+      build-base \
+      openssl-dev \
+      libffi-dev \
+      libxml2-dev \
+      libxslt-dev \
+# Updating pip itself before installing packages from requirements.txt
+ && pip install --no-cache-dir pip setuptools \
+# Installing pip packages from requirements.txt
+ && pip install --no-cache-dir -r requirements.txt \
+ && cd /tmp/utils \
+# Removing release version of scutils package
+ && pip uninstall scutils -y \
+# Installing development version of scutils package
+ && python setup.py install \
+ && cd /usr/src/app \
+# Removing build dependencies leaving image layer clean and neat
+ && apk del build-dependencies
+
+
+# move codebase over
+COPY crawler /usr/src/app
+
+WORKDIR /usr/src/app
+
+# override settings via localsettings.py
+COPY docker/crawler/settings.py /usr/src/app/crawling/localsettings.py
+
+# copy testing script into container
+COPY docker/run_docker_tests.sh /usr/src/app/run_docker_tests.sh
+
+# set up environment variables
+
+# run command
+CMD ["scrapy", "runspider", "crawling/spiders/link_spider.py"]

docker/kafka-monitor/Dockerfile

@@ -7,7 +7,7 @@ RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # install requirements
-COPY requirements.txt /usr/src/app/
+COPY kafka-monitor/requirements.txt /usr/src/app/
 RUN pip install --no-cache-dir -r requirements.txt
 
 # move codebase over

docker/kafka-monitor/Dockerfile.py2alpine

@@ -0,0 +1,51 @@
+FROM python:2.7.12-alpine
+MAINTAINER Madison Bahmer <madison.bahmer@istresearch.com>
+
+
+# copy kafka-monitor own requirements.txt with its dependencies
+COPY kafka-monitor/requirements.txt /usr/src/app/
+# scutils dev version install (not in pip yet)
+# due utils folder located above Dockerfile folder it can not be used
+# for Docker Hub Automated Build feature (but should, see wiki notes)
+COPY utils /tmp/utils
+
+# Combine run command to create single intermeiate image layer
+# This MANDATORY because developments dependencies are huge.
+RUN mkdir -p /usr/src/app \
+ && cd /usr/src/app \
+# Installing runtime dependencies
+ && apk --no-cache add \
+      curl \
+# Installing buildtime dependencies. They will be removed at end of this
+# commands sequence.
+ && apk --no-cache add --virtual build-dependencies \
+      build-base \
+# Updating pip itself before installing packages from requirements.txt
+ && pip install --no-cache-dir pip setuptools \
+# Installing pip packages from requirements.txt
+ && pip install --no-cache-dir -r requirements.txt \
+ && cd /tmp/utils \
+# Removing release version of scutils package
+ && pip uninstall scutils -y \
+# Installing development version of scutils package
+ && python setup.py install \
+ && cd /usr/src/app \
+# Removing build dependencies leaving image layer clean and neat
+ && apk del build-dependencies
+
+# move codebase over
+COPY kafka-monitor /usr/src/app
+
+
+WORKDIR /usr/src/app
+
+# override settings via localsettings.py
+COPY docker/kafka-monitor/settings.py /usr/src/app/localsettings.py
+
+# copy testing script into container
+COPY docker/run_docker_tests.sh /usr/src/app/run_docker_tests.sh
+
+# set up environment variables
+
+# run command
+CMD ["python", "kafka_monitor.py", "run"]

docker/redis-monitor/Dockerfile

@@ -7,7 +7,7 @@ RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # install requirements
-COPY requirements.txt /usr/src/app/
+COPY redis-monitor/requirements.txt /usr/src/app/
 RUN pip install --no-cache-dir -r requirements.txt
 
 # move codebase over

docker/redis-monitor/Dockerfile.py2alpine

@@ -0,0 +1,50 @@
+FROM python:2.7.12-alpine
+MAINTAINER Madison Bahmer <madison.bahmer@istresearch.com>
+
+# copy redis-monitor own requirements.txt with its dependencies
+COPY redis-monitor/requirements.txt /usr/src/app/
+# scutils dev version install (not in pip yet)
+# due utils folder located above Dockerfile folder it can not be used
+# for Docker Hub Automated Build feature (but should, see wiki notes)
+COPY utils /tmp/utils
+
+# Combine run command to create single intermeiate image layer
+# This MANDATORY because developments dependencies are huge.
+RUN mkdir -p /usr/src/app \
+ && cd /usr/src/app \
+# Installing runtime dependencies
+ && apk --no-cache add \
+      curl \
+# Installing buildtime dependencies. They will be removed at end of this
+# commands sequence.
+ && apk --no-cache add --virtual build-dependencies \
+      build-base \
+# Updating pip itself before installing packages from requirements.txt
+ && pip install --no-cache-dir pip setuptools \
+# Installing pip packages from requirements.txt
+ && pip install --no-cache-dir -r requirements.txt \
+ && cd /tmp/utils \
+# Removing release version of scutils package
+ && pip uninstall scutils -y \
+# Installing development version of scutils package
+ && python setup.py install \
+ && cd /usr/src/app \
+# Removing build dependencies leaving image layer clean and neat
+ && apk del build-dependencies
+
+# move codebase over
+COPY redis-monitor /usr/src/app
+
+
+WORKDIR /usr/src/app
+
+# override settings via localsettings.py
+COPY docker/redis-monitor/settings.py /usr/src/app/localsettings.py
+
+# copy testing script into container
+COPY docker/run_docker_tests.sh /usr/src/app/run_docker_tests.sh
+
+# set up environment variables
+
+# run command
+CMD ["python", "redis_monitor.py"]

docker/run_docker_tests.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
 # Script for testing Scrapy Cluster when inside a docker container
 # Will run both the offline and online tests for the particular container,

kafka-monitor/requirements.txt

@@ -0,0 +1,23 @@
+ConcurrentLogHandler==0.9.1
+funcsigs==1.0.2
+functools32==3.2.3.post2
+future==0.15.2
+idna==2.1
+jsonschema==2.5.1
+kafka-python==1.3.1
+kazoo==2.2.1
+mock==2.0.0
+nose==1.3.7
+pbr==1.10.0
+python-json-logger==0.1.5
+python-redis-lock==3.1.0
+PyYAML==3.12
+redis==2.10.5
+requests==2.11.1
+requests-file==1.4
+retrying==1.3.3
+scutils==1.1.0
+six==1.10.0
+testfixtures==4.10.0
+tldextract==2.0.1
+ujson==1.35

redis-monitor/requirements.txt

@@ -0,0 +1,17 @@
+ConcurrentLogHandler==0.9.1
+funcsigs==1.0.2
+future==0.15.2
+kafka-python==1.3.1
+kazoo==2.2.1
+mock==2.0.0
+nose==1.3.7
+pbr==1.10.0
+python-json-logger==0.1.5
+python-redis-lock==3.1.0
+PyYAML==3.12
+redis==2.10.5
+retrying==1.3.3
+scutils==1.1.0
+six==1.10.0
+testfixtures==4.10.0
+ujson==1.35

requirements.txt

@@ -1,3 +1,5 @@
+# this file is deprecated and will be removed
+
 cffi==1.6.0
 characteristic==14.3.0
 ConcurrentLogHandler==0.9.1

travis/docker.sh

@@ -3,24 +3,20 @@
 set -e
 
 # Build Dockerfiles for Scrapy Cluster
-sudo docker build --rm=true --file docker/kafka-monitor/Dockerfile --tag=istresearch/scrapy-cluster:kafka-monitor-test .
-sudo docker build --rm=true --file docker/redis-monitor/Dockerfile --tag=istresearch/scrapy-cluster:redis-monitor-test .
-sudo docker build --rm=true --file docker/crawler/Dockerfile --tag=istresearch/scrapy-cluster:crawler-test .
+sudo docker build --rm=true --file docker/kafka-monitor/$dockerfile_name --tag=istresearch/scrapy-cluster:kafka-monitor-test .
+sudo docker build --rm=true --file docker/redis-monitor/$dockerfile_name --tag=istresearch/scrapy-cluster:redis-monitor-test .
+sudo docker build --rm=true --file docker/crawler/$dockerfile_name --tag=istresearch/scrapy-cluster:crawler-test .
 
 # run docker compose up for docker tests
 sudo docker-compose -f travis/docker-compose.test.yml up -d
 
-# cat kafka logs to check things are working
-sudo docker-compose ps
-sudo docker-compose logs kafka_monitor
-sudo docker-compose logs kafka
-
+# waiting 10 secs for fully operational cluster
 sleep 10
 
 # run docker unit and integration tests for each component
-sudo docker exec -it travis_kafka_monitor_1 env TERM=xterm /bin/bash -c "./run_docker_tests.sh"
-sudo docker exec -it travis_redis_monitor_1 env TERM=xterm /bin/bash -c "./run_docker_tests.sh"
-sudo docker exec -it travis_crawler_1 env TERM=xterm /bin/bash -c "./run_docker_tests.sh"
+sudo docker-compose -f travis/docker-compose.test.yml exec kafka_monitor ./run_docker_tests.sh
+sudo docker-compose -f travis/docker-compose.test.yml exec redis_monitor ./run_docker_tests.sh
+sudo docker-compose -f travis/docker-compose.test.yml exec crawler ./run_docker_tests.sh
 
 # spin down compose
 sudo docker-compose -f travis/docker-compose.test.yml down
@@ -29,9 +25,14 @@ sudo docker-compose -f travis/docker-compose.test.yml down
 
 if [ "$TRAVIS_BRANCH" == "dev" ]; then
     # build 'dev' docker images for dockerhub
-    sudo docker build --rm=true --file docker/kafka-monitor/Dockerfile --tag=istresearch/scrapy-cluster:kafka-monitor-dev .
-    sudo docker build --rm=true --file docker/redis-monitor/Dockerfile --tag=istresearch/scrapy-cluster:redis-monitor-dev .
-    sudo docker build --rm=true --file docker/crawler/Dockerfile --tag=istresearch/scrapy-cluster:crawler-dev .
+    sudo docker build --rm=true --file docker/kafka-monitor/$dockerfile_name --tag=istresearch/scrapy-cluster:kafka-monitor-$docker_tag_suffix .
+    sudo docker build --rm=true --file docker/redis-monitor/$dockerfile_name --tag=istresearch/scrapy-cluster:redis-monitor-$docker_tag_suffix .
+    sudo docker build --rm=true --file docker/crawler/$dockerfile_name --tag=istresearch/scrapy-cluster:crawler-$docker_tag_suffix .
+
+    # remove 'test' images
+    sudo docker rmi istresearch/scrapy-cluster:kafka-monitor-test
+    sudo docker rmi istresearch/scrapy-cluster:redis-monitor-test
+    sudo docker rmi istresearch/scrapy-cluster:crawler-test
 
     # log into docker
     sudo docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"