Add privacyscanner to Ansible scripts

This commits adds neccessary steps to install privacyscanner and its dependencies (e.g. Google Chrome). It also removes steps to install OpenWPM as it was replaced by privacyscanner and notes that in the `README.md`. Also some minor improvements were made: - replace internal Git repository with GitHub - add missing entries to `settings.py` template - add `sudo` as dependency as it was needed during the install - use Bash as default shell for user `privacyscore`
PrivacyScore · Sep 25, 2018 · 289f72a · 289f72a
1 parent f11cd8d
commit 289f72a
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 98 deletions.
diff --git a/README.md b/README.md
@@ -14,7 +14,7 @@ At the moment, the code should be considered beta quality. To try the system out
 PrivacyScore relies on the following libraries and frameworks:
 
 - [Django](https://www.djangoproject.com/) (BSD)
-- [OpenWPM](https://github.com/citp/OpenWPM) (GPLv3)
+- [Privacyscanner](https://github.com/PrivacyScore/privacyscanner) (MIT + GPLv3)
 - [testssl.sh](https://github.com/drwetter/testssl.sh) (GPLv2)¹
 - [Celery](http://www.celeryproject.org/) (BSD)
 - [adblockparser](https://github.com/scrapinghub/adblockparser) (MIT)

diff --git a/ansible/deploy_slave.yml b/ansible/deploy_slave.yml
@@ -4,25 +4,18 @@
   vars:
     ansible_ssh_pipelining: yes
   tasks:
-    - name: Add contrib to sources.list
-      blockinfile:
-        path: /etc/apt/sources.list
-        block: |
-          deb http://192.168.122.1:3142/ftp.de.debian.org/debian/ stretch main contrib
-          deb-src http://192.168.122.1:3142/ftp.de.debian.org/debian/ stretch main contrib
     - name: Install apt packages
       apt: name={{ item }} update_cache=yes
       with_items:
         - build-essential
-        - firefox-esr
-        - flashplugin-nonfree
         - git
         - libboost-python-dev
         - libffi-dev
         - libjpeg-dev
         - libleveldb1v5  # v5 on stretch
         - libleveldb-dev
         - libpq-dev
+        - libre2-dev
         - libssl-dev
         - libxml2-dev
         - libxslt-dev
@@ -31,8 +24,12 @@
         - python3-dev
         - python3-six
         - python-dev
+        - sudo
         - virtualenv
         - xvfb
+    - name: Download and install Google Chrome for privacyscanner
+      apt:
+        deb: https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
     - name: Enable NTP service
       service:
         name: ntp
@@ -52,16 +49,17 @@
         owner: root
         group: root
         mode: 0400
-    - name: Clone git repository
+    - name: Clone privacyscore git repository
       git:
-        repo: gogs@svs.informatik.uni-hamburg.de:svs/privacyscore.git
+        repo: https://github.com/PrivacyScore/PrivacyScore.git
         dest: /opt/privacyscore
         version: master
         accept_hostkey: yes
     - name: Create privacyscore user
       user:
         name: privacyscore
         home: /opt/privacyscore
+        shell: /bin/bash
         system: yes
     - name: Set owner of privacyscore directory
       file:
@@ -126,7 +124,7 @@
       become: yes
       become_user: privacyscore
       git:
-        repo: gogs@svs.informatik.uni-hamburg.de:svs/privacyscore.git
+        repo: https://github.com/PrivacyScore/PrivacyScore.git
         dest: /opt/privacyscore
         version: master
         accept_hostkey: yes
diff --git a/ansible/templates/settings.py b/ansible/templates/settings.py
@@ -44,6 +44,7 @@
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'rest_framework',
+    'widget_tweaks',
 ]
 
 MIDDLEWARE = [
@@ -105,6 +106,12 @@
 # Internationalization
 # https://docs.djangoproject.com/en/1.11/topics/i18n/
 
+from django.utils.translation import gettext_lazy as _
+LANGUAGES = [
+    ('de', _('German')),
+    ('en', _('English')),
+]
+
 LANGUAGE_CODE = 'en-us'
 
 TIME_ZONE = 'UTC'

diff --git a/ansible/update_hosts.yml b/ansible/update_hosts.yml
@@ -4,7 +4,7 @@
   vars:
     ansible_ssh_pipelining: yes
   tasks:
-    - name: Refresh git repository
+    - name: Refresh PrivacyScore git repository
       become: yes
       become_user: privacyscore
       git:
@@ -87,7 +87,7 @@
               # release based on the git info.
               'release': raven.fetch_git_sha('/opt/privacyscore'),
           }
-    - name: Install pip requirements
+    - name: Install PrivacyScore pip requirements
       become: yes
       become_user: privacyscore
       pip:
@@ -103,6 +103,29 @@
       environment:
         PATH: /opt/privacyscore/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
         VIRTUAL_ENV: /opt/privacyscore/.pyenv
+    - name: Refresh privacyscanner git repository
+      become: yes
+      become_user: privacyscore
+      git:
+        repo: https://github.com/PrivacyScore/privacyscanner.git
+        dest: /opt/privacyscore/privacyscanner
+        version: master
+    - name: Install privacyscanner pip requirements
+      become: yes
+      become_user: privacyscore
+      pip:
+        editable: yes
+        name: /opt/privacyscore/privacyscanner
+      environment:
+        PATH: /opt/privacyscore/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+        VIRTUAL_ENV: /opt/privacyscore/.pyenv
+    - name: Update privacyscanner dependencies
+      become: yes
+      become_user: privacyscore
+      command: privacyscanner update_dependencies
+      environment:
+        PATH: /opt/privacyscore/.pyenv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+        VIRTUAL_ENV: /opt/privacyscore/.pyenv
     - name: Migrate database
       when: is_master
       become: yes
@@ -151,42 +174,6 @@
         repo: https://github.com/drwetter/testssl.sh.git
         dest: /opt/privacyscore/tests/vendor/testssl.sh
         version: 47409a32d687355739543890c1ddc70a8539d071
-    - name: Refresh vendor/OpenWPM
-      when: is_slave
-      become: yes
-      become_user: privacyscore
-      git:
-        repo: https://github.com/citp/OpenWPM
-        dest: /opt/privacyscore/tests/vendor/OpenWPM
-        version: 99da2d91897c2ddcb128a0e133cccc09c4f14996
-    - name: Create vendor/EasyList if it does not exist
-      file:
-        path: /opt/privacyscore/tests/vendor/EasyList/
-        owner: privacyscore
-        group: privacyscore
-        mode: 0700
-        state: directory
-    - name: Download EasyList
-      get_url:
-        url: https://easylist.to/easylist/easylist.txt
-        owner: privacyscore
-        group: privacyscore
-        dest: /opt/privacyscore/tests/vendor/EasyList/easylist.txt
-        force: yes
-    - name: Download EasyPrivacy
-      get_url:
-        url: https://easylist.to/easylist/easyprivacy.txt
-        owner: privacyscore
-        group: privacyscore
-        dest: /opt/privacyscore/tests/vendor/EasyList/easyprivacy.txt
-        force: yes
-    - name: Download Fanboy Anti-Social-Media
-      get_url:
-        url: https://easylist.to/easylist/fanboy-annoyance.txt
-        owner: privacyscore
-        group: privacyscore
-        dest: /opt/privacyscore/tests/vendor/EasyList/fanboy-annoyance.txt
-        force: yes
     - name: Create vendor/HSTSPreload if it does not exist
       file:
         path: /opt/privacyscore/tests/vendor/HSTSPreload/

diff --git a/install-tests.sh b/install-tests.sh
@@ -8,59 +8,16 @@ touch tests/vendor/__init__.py
 mkdir -p tests/vendor/geoip
 wget --quiet -O- http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz | gunzip > tests/vendor/geoip/GeoLite2-Country.mmdb
 
-# openwpm
-# We cannot use --depth 1 very nicely, because we cannot simply
-# clone a specific commit only :( We cannot even fetch a commit unless
-# there is a ref on the remote. Git is not as clever as, say,
-# mercurial in this regard.  So we have to download the whole
-# repository all the time.
-git clone https://github.com/citp/OpenWPM tests/vendor/OpenWPM
-pushd tests/vendor/OpenWPM > /dev/null
-# Hrm, why exactly do we have the versions here again?
-# We also have versions in the Ansible files.
-git checkout 99da2d91897c2ddcb128a0e133cccc09c4f14996
-
-# create openwpm-specific virtualenv
-deactivate || true
-virtualenv --python $(which python2) .pyenv
-. .pyenv/bin/activate
-
-pip install -U pip
-pip install six tldextract packaging appdirs
-# This is a hack for making openwpm work with this ancient Debian.
-# OpenWPM pulls any version of plyvel. With the 1.0.0 release,
-# it requires libleveldb as of 1.20. Debian stale does not ship that.
-# Previous plyvel releases did not require that new version.
-pip install "plyvel < 1.0.0"
-# Hence we cannot run the --upgrade, because it would pull the newer version
-# pip install -U -r requirements.txt
-pip install -r requirements.txt
-
-# Install specific version of Firefox known to work well with the selenium version above
-if [ $(uname -m) == 'x86_64' ]; then
-  echo Downloading 64-bit Firefox
-  wget https://ftp.mozilla.org/pub/firefox/releases/45.9.0esr/linux-x86_64/en-US/firefox-45.9.0esr.tar.bz2
-else
-  echo Downloading 32-bit Firefox
-  wget https://ftp.mozilla.org/pub/firefox/releases/45.9.0esr/linux-i686/en-US/firefox-45.9.0esr.tar.bz2
-fi
-tar jxf firefox*.tar.bz2
-rm -rf firefox-bin
-mv firefox firefox-bin
-rm firefox*.tar.bz2
-
-deactivate
+# privacyscanner
+git clone https://github.com/PrivacyScore/privacyscanner.git
+pushd privacyscanner
+pip install -e .
+privacyscanner update_dependencies
 popd
 
 # testssl.sh
 git clone https://github.com/drwetter/testssl.sh.git tests/vendor/testssl.sh
 
-# EasyList
-mkdir tests/vendor/EasyList
-pushd tests/vendor/EasyList
-wget https://easylist.to/easylist/easyprivacy.txt https://easylist.to/easylist/easylist.txt https://easylist.to/easylist/fanboy-annoyance.txt
-popd
-
 # HSTS Preload List
 mkdir tests/vendor/HSTSPreload
 pushd tests/vendor/HSTSPreload