Merge pull request #12 from datasciencebr/irio-facebook-messenger

Make code ready to Digital Ocean

.env.example

@@ -1,3 +1,4 @@
+FLASK_DEBUG=1
 TWITTER_ACCESS_TOKEN_KEY=YOUR_ACCESS_TOKEN_KEY
 TWITTER_ACCESS_TOKEN_SECRET=YOUR_ACCESS_TOKEN_SECRET
 TWITTER_CONSUMER_KEY=YOUR_CONSUMER_KEY

Dockerfile

@@ -1,20 +1,13 @@
 FROM python:3.6.1
 
-# RUN apt-get update \
-#     && apt-get install -y --no-install-recommends \
-#         postgresql-client \
-#     && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /usr/src/app
-COPY requirements.txt ./
-RUN pip install -U -r requirements.txt
-
 RUN mkdir rosie
 COPY rosie/config.ini.example ./config.ini
 COPY rosie/requirements.txt ./rosie
 RUN pip install -r rosie/requirements.txt
 
-COPY . .
+WORKDIR /usr/src/app
+COPY requirements.txt ./
+RUN pip install -r requirements.txt
 
 RUN useradd -ms /bin/bash serenata_de_amor
 RUN chown -hR serenata_de_amor .

README.md

@@ -12,7 +12,7 @@ Follow [@RosieDaSerenata](https://twitter.com/RosieDaSerenata) to get Brazilian
 Install [Docker](https://www.docker.com) and [Docker Compose](https://docs.docker.com/compose/).
 
 ```console
-$ git clone --recursive git@github.com:Irio/whistleblower.git
+$ git clone --recursive git@github.com:datasciencebr/whistleblower.git
 $ cd whistleblower
 $ cp .env.example .env
 $ docker-compose build
@@ -21,7 +21,6 @@ $ docker-compose build
 ## Running
 
 ```console
-$ docker-compose run worker python whistleblower/get_dataset.py
 $ docker-compose run worker python rosie/rosie.py run chamber_of_deputies data
 $ docker-compose up
 ```

bootstrap_production.sh

@@ -0,0 +1,39 @@
+# Configure SSH access
+# source: https://www.digitalocean.com/community/tutorials/initial-server-setup-with-ubuntu-16-04
+#
+# ----------
+
+useradd -ms /bin/bash serenata_de_amor
+git clone --recursive https://github.com/datasciencebr/whistleblower.git /home/serenata_de_amor/whistleblower
+chown -hR serenata_de_amor /home/serenata_de_amor/whistleblower
+
+# Setup auto restart services after VM restart
+cat >/home/serenata_de_amor/server.sh <<EOL
+touch /home/serenata_de_amor/running.txt
+
+sudo -u serenata_de_amor bash << EOF
+
+cd /home/serenata_de_amor/whistleblower
+nohup docker-compose -f docker-compose.yml up &
+
+EOF
+EOL
+(crontab -l ; echo "@reboot /home/serenata_de_amor/server.sh") | sort - | uniq - | crontab -
+
+# Install Docker
+# source: https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-16-04
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+sudo apt-get update
+apt-cache policy docker-ce
+sudo apt-get install -y docker-ce
+sudo systemctl status docker
+sudo usermod -aG docker serenata_de_amor
+
+# Install Docker Compose
+# source: https://www.digitalocean.com/community/tutorials/how-to-install-docker-compose-on-ubuntu-16-04
+sudo curl -o /usr/local/bin/docker-compose -L "https://github.com/docker/compose/releases/download/1.11.2/docker-compose-$(uname -s)-$(uname -m)"
+sudo chmod +x /usr/local/bin/docker-compose
+
+# Missing: .env file and environment-specific docker-compose.yml

config.ini.example

@@ -1,7 +1,7 @@
 [Amazon]
 Bucket: serenata-de-amor-data
 AccessKey: YOUR_ACCESS_KEY
-Region: s3-sa-east-1
+Region: sa-east-1
 SecretKey: YOUR_SECRET_KEY
 
 [Foursquare]

docker-compose.override.yml

@@ -0,0 +1,9 @@
+version: "3"
+services:
+  mongo:
+    ports:
+      - 27017:27017
+
+  rabbitmq:
+    ports:
+      - 5672:5672

docker-compose.yml

@@ -1,18 +1,21 @@
 version: "3"
 services:
   mongo:
-    image: mongo:3.5.7
-    ports:
-      - 27017:27017
+    image: mongo:3.5.9
+    restart: always
+    expose:
+      - "27017"
 
   rabbitmq:
-    image: rabbitmq:3.6.9
-    ports:
-      - 5672:5672
+    image: rabbitmq:3.6.10
+    restart: always
+    expose:
+      - "5672"
 
   worker:
     build: .
     command: celery -A whistleblower.tasks worker -B
+    restart: always
     depends_on:
       - mongo
       - rabbitmq
@@ -24,5 +27,6 @@ services:
       - MONGO_URL=mongodb://mongo:27017/
     volumes:
       - ./data:/usr/src/app/data
-      - ./whistleblower:/usr/src/app/whistleblower
+      - ./rosie:/usr/src/app/rosie
       - ./tests:/usr/src/app/tests
+      - ./whistleblower:/usr/src/app/whistleblower

requirements.txt

@@ -1,8 +1,8 @@
-amqp==2.1.4
-celery==4.0.2
+amqp==2.2.2
+celery==4.1.0
 ipdb==0.10.3
-ipython==6.1.0
-pandas==0.19.2
-pymongo==3.4.0
-python-twitter==3.2.1
-serenata-toolbox==12.0.4
+pandas==0.21.0
+pymongo==3.5.1
+python-twitter==3.3
+requests==2.18.4
+serenata-toolbox==12.2.2

tests/targets/test_twitter.py

@@ -17,7 +17,7 @@ def setUp(self):
     def test_profiles(self):
         self.subject = Twitter(api=self.api,
                                database=self.database,
-                               profiles_file='tests/fixtures/twitter_profiles.csv')
+                               profiles_file='tests/fixtures/congresspeople-social-accounts.csv')
         self.assertIsInstance(self.subject.profiles(), pd.DataFrame)
 
     def test_posted_reimbursements(self):
@@ -36,7 +36,7 @@ def test_follow_congresspeople(self, logging_mock):
             ['DepRodrigomaia', None],
             [None, None]
         ], columns=['twitter_profile', 'secondary_twitter_profile'])
-        self.subject.profiles = profiles
+        self.subject._profiles = profiles
         calls = [
             mock.call.CreateFriendship(screen_name='DepEduardoCunha'),
             mock.call.CreateFriendship(screen_name='DepEduardoCunha2'),
@@ -108,7 +108,7 @@ def setUp(self):
         self.reimbursement = {
             'congressperson_name': 'Eduardo Cunha',
             'document_id': 10,
-            'state_x': 'RJ',
+            'state': 'RJ',
             'twitter_profile': 'DepEduardoCunha',
         }
         self.subject = Post(self.reimbursement,

tests/test_queue.py

@@ -0,0 +1,45 @@
+from unittest import TestCase, mock
+
+import pandas as pd
+
+from whistleblower.queue import Queue as subject_class
+
+
+class TestQueue(TestCase):
+    def setUp(self):
+        self.database = mock.MagicMock()
+        self.subject = subject_class(self.database)
+
+    @mock.patch('whistleblower.queue.Suspicions')
+    @mock.patch('whistleblower.queue.Twitter')
+    def test_update(self, twitter_mock, suspicions_mock):
+        suspicions = pd.DataFrame([pd.Series({'document_id': 1})])
+        twitter_mock.return_value.post_queue.return_value.sample.return_value = suspicions
+        self.subject.update()
+        self.database.queue.delete_many.assert_called_once_with({})
+        self.database.queue.create_index.assert_called_once_with('document_id', unique=True)
+        self.database.queue.insert_many.assert_called_once_with(
+            list(self.subject.remaining_posts()), ordered=False)
+
+    @mock.patch('whistleblower.queue.whistleblower.tasks.publish_reimbursement')
+    def test_process(self, publish_reimbursement_mock):
+        reimbursement = {'document_id': 10}
+        self.database.queue.find_one_and_delete.return_value = reimbursement
+        self.subject.process()
+        publish_reimbursement_mock.assert_called_once_with(reimbursement)
+
+    @mock.patch('whistleblower.queue.Suspicions')
+    @mock.patch('whistleblower.queue.Twitter')
+    def test_remaining_posts(self, twitter_mock, suspicions_mock):
+        suspicions = pd.DataFrame([pd.Series({'document_id': i}) for i in range(1, 5)])
+        suspicions_mock.return_value.all.return_value = suspicions
+        twitter_mock.return_value.post_queue.return_value.sample.return_value = suspicions.head(2)
+        response = [post for post in self.subject.remaining_posts()]
+        self.assertEqual(suspicions.iloc[0].to_dict(), response[0])
+        self.assertEqual(suspicions.iloc[1].to_dict(), response[1])
+
+    @mock.patch('whistleblower.queue.Suspicions')
+    def test_reimbursements(self, suspicions_mock):
+        suspicions = [{'document_id': 10}]
+        suspicions_mock.return_value.all.return_value = suspicions
+        self.assertEqual(suspicions, self.subject.reimbursements())

web/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.6.1
+
+WORKDIR /usr/src/app
+COPY requirements.txt ./
+RUN pip install -r requirements.txt
+
+RUN useradd -ms /bin/bash serenata_de_amor
+RUN chown -hR serenata_de_amor .
+USER serenata_de_amor
+
+COPY server.py ./
+CMD flask run --host=0.0.0.0

web/requirements.txt

@@ -0,0 +1,2 @@
+flask==0.12.2
+pymongo==3.4.0

web/server.py

@@ -0,0 +1,22 @@
+import http.client
+import os
+
+from flask import Flask
+from pymongo import MongoClient
+
+MONGO_URL = os.environ.get('MONGO_URL', 'mongodb://mongo:27017/')
+MONGO_DATABASE = os.environ.get('MONGO_DATABASE', 'whistleblower')
+DATABASE = MongoClient(MONGO_URL)[MONGO_DATABASE]
+app = Flask(__name__)
+
+
+@app.route('/')
+def hello_world():
+    return 'Hello, World!'
+
+
+# @app.route('/facebook_webhook', methods=['POST'])
+@app.route('/facebook_webhook')
+def facebook_webhook():
+    DATABASE.facebook_webhook.insert(request.form)
+    return ('', http.client.NO_CONTENT)