Merge pull request #232 from OpenDataPolicingNC/docker-setup

Docker setup

Dockerfile

@@ -0,0 +1,45 @@
+FROM python:3.4
+ENV PYTHONUNBUFFERED 1
+
+# Install packages for postgres, python dev for some project packages, and rsync for some fabric commands:
+RUN apt-get update && \
+  apt-get install -y \
+    binutils \
+    postgresql-client \
+    rsync \
+    gdal-bin
+
+# Fabric for deploys:
+RUN pip install -U pip
+RUN pip install PyYAML fabric3 paramiko pycrypto ecdsa
+
+RUN mkdir /code
+WORKDIR /code
+
+ADD requirements /code/requirements
+RUN pip install -r requirements/dev.txt
+
+# nvm environment variables
+ENV NVM_DIR /usr/local/nvm
+RUN mkdir -p $NVM_DIR
+ENV NODE_VERSION 6.14.4
+
+# install nvm
+# https://github.com/creationix/nvm#install-script
+RUN wget -qO- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash
+
+# install node and npm
+RUN . $NVM_DIR/nvm.sh \
+    && nvm install $NODE_VERSION \
+    && nvm alias default $NODE_VERSION \
+    && nvm use default
+
+# add node and npm to path so the commands are available
+ENV NODE_PATH $NVM_DIR/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+
+# confirm installation
+RUN node -v
+RUN npm -v
+
+RUN npm install -g gulp

docker-compose.yml

@@ -0,0 +1,33 @@
+version: '3'
+
+services:
+  db:
+    image: mdillon/postgis:9.4-alpine
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+  web:
+    build: .
+    command: npm run dev -- --address=0.0.0.0 --port=8000
+    environment:
+      DJANGO_SETTINGS_MODULE: traffic_stops.settings.local
+      PGDATABASE: traffic_stops
+      PGUSER: postgres
+      PGHOST: db
+      DATABASE_URL: postgis://postgres@db:5432/traffic_stops
+      DATABASE_URL_NC: postgis://postgres@db:5432/traffic_stops_nc
+      DATABASE_URL_MD: postgis://postgres@db:5432/traffic_stops_md
+      DATABASE_URL_IL: postgis://postgres@db:5432/traffic_stops_il
+      VIRTUAL_ENV: /usr/local  # gulpfile expects Python to be in a virtualenv
+    volumes:
+      # mount local repository into the container
+      - .:/code:delegated
+      # If the developer wants to deploy inside a docker container, expose their
+      # SSH keys to the container:
+      - ~/.ssh:/root/.ssh
+    ports:
+      - "8003:8000"
+    depends_on:
+      - db
+
+volumes:
+  pgdata:

docs/dev-setup.rst

@@ -72,6 +72,7 @@ Create the Postgres database and run the initial syncdb/migrate::
     (opendatapolicing)$ createdb -E UTF-8 traffic_stops_il
     (opendatapolicing)$ ./migrate_all_dbs.sh
 
+
 Development
 -----------
 
@@ -93,3 +94,85 @@ This is a multi-database project.  Whenever you have unapplied migrations,
 either added locally or via an update from the source repository, the
 migrations need to be applied to all databases by running the
 ``./migrate_all_dbs.sh`` command.
+
+
+Docker
+======
+
+You can use the provided ``docker-compose`` environment to create a local development environment.
+See previous section for more detailed instructions about how this project is configured.
+
+Setup your local development settings::
+
+  cp traffic_stops/settings/local.example.py traffic_stops/settings/local.py
+  # uncomment lines below "UNCOMMENT BELOW IF USING DOCKER SETUP" in local.py
+  echo "DJANGO_SETTINGS_MODULE=traffic_stops.settings.local" > .env
+
+For basic setup, run the following commands::
+
+  docker-compose up -d db  # start the PostgreSQL container in the background
+  docker-compose build web  # build the container (can take a while)
+  docker-compose run --rm web createdb -E UTF-8 traffic_stops
+  docker-compose run --rm web createdb -E UTF-8 traffic_stops_nc
+  docker-compose run --rm web createdb -E UTF-8 traffic_stops_md
+  docker-compose run --rm web createdb -E UTF-8 traffic_stops_il
+  docker-compose run --rm web ./migrate_all_dbs.sh
+
+Run ``npm install`` inside the ``web`` container (you'll need to do this for any update to
+``package.json``)::
+
+  rm -rf ./node_modules  # if needed
+  docker-compose run --rm web bash -lc "npm install"
+
+You can now run the web container and tail the logs::
+
+  # start up the dev server, and watch the logs:
+  docker-compose up -d web && docker-compose logs -f web
+
+These are other useful docker-compose commands::
+
+  # explicitly execute runserver in the foreground (for breakpoints):
+  docker-compose stop web
+  docker-compose run --rm --service-ports web python manage.py runserver 0.0.0.0:8000
+
+Visit http://localhost:8003/ in your browser.
+
+
+Restore Production Data
+-----------------------
+
+The data import process for each state can take a long time. You can load the production data using
+the following steps:
+
+First download a dump (in this case, NC) of the database::
+
+  ssh opendatapolicing.com 'sudo -u postgres pg_dump -Fc -Ox traffic_stops_nc_production' > traffic_stops_nc_production.pgdump
+
+Now run ``pg_restore`` within the ``web`` container::
+
+  docker-compose stop web  # free up connections to the DB
+  docker-compose run --rm web dropdb traffic_stops_nc
+  docker-compose run --rm web createdb -E UTF-8 traffic_stops_nc
+  docker-compose run --rm web pg_restore -Ox -d traffic_stops_nc traffic_stops_nc_production.pgdump
+  rm traffic_stops_nc_production.pgdump  # so it doesn't get built into the container
+
+You can also load the primary DB with user accounts and state statistics::
+
+  ssh opendatapolicing.com 'sudo -u postgres pg_dump -Fc -Ox traffic_stops_production' > traffic_stops_production.pgdump
+  docker-compose stop web  # free up connections to the DB
+  docker-compose run --rm web dropdb traffic_stops
+  docker-compose run --rm web createdb -E UTF-8 traffic_stops
+  docker-compose run --rm web pg_restore -Ox -d traffic_stops traffic_stops_production.pgdump
+  rm traffic_stops_production.pgdump  # so it doesn't get built into the container
+
+
+Deployment
+----------
+
+You can run a deployment from within a docker container using the following commands::
+
+  docker-compose run --rm web /bin/bash
+  eval $(ssh-agent)
+  ssh-add ~/.ssh/YOUR_KEY
+
+  fab -u YOUR_USER staging salt:"test.ping"

il/templates/il/agency_detail.html

@@ -109,14 +109,14 @@ <h4 class="graph-label">
 {% block stop-count-display %}
   <div class="row">
     <div class="col-md-12">
-    <h3>Departmental Stop Count</h3>
+      <h3>Departmental Stop Count</h3>
 
-    <p class="help-block">
-      This graph displays the number of traffic stops broken down by stop
-      purpose and ethnicity. Adjusting the drop down menu will display the
-      individual stop counts relative to ethnic groups on a year-by-year basis.
-    </p>
-  </div>
+      <p class="help-block">
+        This graph displays the number of traffic stops broken down by stop
+        purpose and ethnicity. Adjusting the drop down menu will display the
+        individual stop counts relative to ethnic groups on a year-by-year basis.
+      </p>
+    </div>
 
     <div class="col-md-10">
       <div role="tabpanel">

nc/api.py

@@ -10,6 +10,7 @@
 from rest_framework_extensions.key_constructor.constructors import DefaultObjectKeyConstructor
 
 from nc.models import Agency, Stop
+from nc.models import SEARCH_TYPE_CHOICES as SEARCH_TYPE_CHOICES_TUPLES
 from nc import serializers
 from tsdata.utils import GroupedData
 
@@ -39,6 +40,8 @@
                   'white': 0,
                   'hispanic': 0}
 
+SEARCH_TYPE_CHOICES = dict(SEARCH_TYPE_CHOICES_TUPLES)
+
 
 class QueryKeyConstructor(DefaultObjectKeyConstructor):
     params_query = bits.QueryParamsKeyBit(['officer'])
@@ -76,6 +79,12 @@ def query(self, results, group_by, filter_=None):
                                               stop['purpose'])
                 data['purpose'] = purpose
 
+            if 'search__type' in group_by:
+                data['search_type'] = SEARCH_TYPE_CHOICES.get(
+                    stop['search__type'],
+                    stop['search__type'],
+                )
+
             if 'person__race' in group_by:
                 # The 'Hispanic' ethnicity option is now being aggreggated into its
                 # own race category, and its count excluded from the other counts.
@@ -128,6 +137,23 @@ def searches(self, request, pk=None):
         self.query(results, group_by=('year', 'person__race', 'person__ethnicity'), filter_=q)
         return Response(results.flatten())
 
+    @detail_route(methods=['get'])
+    @cache_response(key_func=query_cache_key_func)
+    def searches_by_type(self, request, pk=None):
+        results = GroupedData(by=('search_type', 'year'), defaults=GROUP_DEFAULTS)
+        q = Q(search__isnull=False)
+        self.query(
+            results,
+            group_by=(
+                'search__type',
+                'year',
+                'person__race',
+                'person__ethnicity',
+            ),
+            filter_=q,
+        )
+        return Response(results.flatten())
+
     @detail_route(methods=['get'])
     @cache_response(key_func=query_cache_key_func)
     def contraband_hit_rate(self, request, pk=None):

nc/prime_cache.py

@@ -9,7 +9,14 @@
 from nc.models import Agency
 
 logger = logging.getLogger(__name__)
-ENDPOINTS = ('stops', 'stops_by_reason', 'use_of_force', 'searches', 'contraband_hit_rate')
+ENDPOINTS = (
+    'stops',
+    'stops_by_reason',
+    'use_of_force',
+    'searches',
+    'searches_by_type',
+    'contraband_hit_rate',
+)
 DEFAULT_CUTOFF_SECS = 4
 
 

nc/templates/nc/agency_detail.html

@@ -126,6 +126,88 @@ <h4 class="graph-label">
   </div>
 {% endblock stop-display %}
 
+{% block search-count-display %}
+<div class="row">
+  <div class="col-md-12">
+  <h3>{% if officer_id %}Officer{% else %}Departmental{% endif %} Search Count</h3>
+
+  <p class="help-block">
+    This graph displays the number of searches broken down by search
+    type and ethnicity. Adjusting the drop down menu will display the
+    individual search counts relative to ethnic groups on a year-by-year basis.
+  </p>
+</div>
+
+  <div class="col-md-10">
+    <div role="tabpanel">
+      <!-- Nav tabs -->
+      <ul class="nav nav-tabs" role="tablist">
+        <li role="presentation" class="active">
+          <a href="#st_line_div" role="tab" data-toggle="tab">Chart</a></li>
+        <li role="presentation">
+          <a href="#st_data" role="tab" data-toggle="tab">Data</a></li>
+      </ul>
+
+      <!-- Tab panes -->
+      <div class="tab-content">
+        <div role="tabpanel" class="tab-pane active" id="st_line_div">
+          <svg id="st_line"></svg>
+        </div>
+        <div role="tabpanel" class="tab-pane" id="st_data">
+        </div>
+      </div>
+    </div>
+
+    <p class="graph-help-block">
+      Drag the cursor over the graph to see the racial breakdown for any
+      given year. Some percentages may be based on low levels of observation.
+      Click the “Data” tab to review the raw stop data.
+    </p>
+  </div>
+</div>
+{% endblock search-count-display %}
+
+{% block stop-count-display %}
+  <div class="row">
+    <div class="col-md-12">
+    <h3>Departmental Stop Count</h3>
+
+    <p class="help-block">
+      This graph displays the number of traffic stops broken down by stop
+      purpose and ethnicity. Adjusting the drop down menu will display the
+      individual stop counts relative to ethnic groups on a year-by-year basis.
+    </p>
+  </div>
+
+    <div class="col-md-10">
+      <div role="tabpanel">
+        <!-- Nav tabs -->
+        <ul class="nav nav-tabs" role="tablist">
+          <li role="presentation" class="active">
+            <a href="#srr_line_div" role="tab" data-toggle="tab">Chart</a></li>
+          <li role="presentation">
+            <a href="#srr_data" role="tab" data-toggle="tab">Data</a></li>
+        </ul>
+
+        <!-- Tab panes -->
+        <div class="tab-content">
+          <div role="tabpanel" class="tab-pane active" id="srr_line_div">
+            <svg id="srr_line"></svg>
+          </div>
+          <div role="tabpanel" class="tab-pane" id="srr_data">
+          </div>
+        </div>
+      </div>
+
+      <p class="graph-help-block">
+        Drag the cursor over the graph to see the racial breakdown for any
+        given year. Some percentages may be based on low levels of observation.
+        Click the “Data” tab to review the raw stop data.
+      </p>
+    </div>
+  </div>
+{% endblock stop-count-display %}
+
 {% block search-rate-display %}
   <div class="row">
     <div class="col-md-12 headline">
@@ -452,13 +534,15 @@ <h4>Longitudinal view of annual use-of-force by race/ethnic composition</h4>
   // Retrieve data from API using data-handlers
   var stop_handler = new NC.StopsHandler({url: "{% url 'nc:agency-api-stops' object.pk %}?officer={{officer_id|urlencode}}"}),
       search_handler = new NC.SearchHandler({url: "{% url 'nc:agency-api-searches' object.pk %}?officer={{officer_id|urlencode}}"}),
+      searches_by_type_handler = new NC.STHandler({url: "{% url 'nc:agency-api-searches-by-type' object.pk %}?officer={{officer_id|urlencode}}"}),
       uf_handler = new NC.UseOfForceHandler({url: "{% url 'nc:agency-api-use-of-force' object.pk %}?officer={{officer_id|urlencode}}"}),
       lhs_handler = new NC.LikelihoodSearchHandler({url: "{% url 'nc:agency-api-stops-by-reason' object.pk %}?officer={{officer_id|urlencode}}"}),
       chr_handler = new NC.ContrabandHitRateHandler({url: "{% url 'nc:agency-api-contraband-hit-rate' object.pk %}?officer={{officer_id|urlencode}}"});
 {% else %}
   // Retrieve data from API using data-handlers
   var stop_handler = new NC.StopsHandler({url: "{% url 'nc:agency-api-stops' object.pk %}"}),
       search_handler = new NC.SearchHandler({url: "{% url 'nc:agency-api-searches' object.pk %}"}),
+      searches_by_type_handler = new NC.STHandler({url: "{% url 'nc:agency-api-searches-by-type' object.pk %}"}),
       uf_handler = new NC.UseOfForceHandler({url: "{% url 'nc:agency-api-use-of-force' object.pk %}"}),
       lhs_handler = new NC.LikelihoodSearchHandler({url: "{% url 'nc:agency-api-stops-by-reason' object.pk %}"}),
       chr_handler = new NC.ContrabandHitRateHandler({url: "{% url 'nc:agency-api-contraband-hit-rate' object.pk %}"});
@@ -472,6 +556,9 @@ <h4>Longitudinal view of annual use-of-force by race/ethnic composition</h4>
 new NC.StopRatioTimeSeries({handler: stop_handler, selector: "#stop_race_line", showEthnicity: showEthnicity});
 new NC.StopsTable({handler: stop_handler, selector: "#stop_race_data", showEthnicity: showEthnicity});
 
+new NC.STTimeSeries({handler: searches_by_type_handler, selector: '#st_line', showEthnicity: showEthnicity});
+new NC.STTable({handler: searches_by_type_handler, selector: '#st_data', showEthnicity: showEthnicity});
+
 new NC.StopSearchTimeSeries({handler: stop_search_handler, selector: "#stop_search_line", showEthnicity: showEthnicity});
 new NC.StopSearchTable({handler: stop_search_handler, selector: "#stop_search_data", showEthnicity: showEthnicity});
 
@@ -489,5 +576,8 @@ <h4>Longitudinal view of annual use-of-force by race/ethnic composition</h4>
 new NC.UseOfForceBarChart({handler: uf_handler, selector: "#use_force_race_line", showEthnicity: showEthnicity});
 new NC.UseOfForceTable({handler: uf_handler, selector: "#use_force_race__data", showEthnicity: showEthnicity});
 
+new NC.SRRTimeSeries({handler: lhs_handler, selector: '#srr_line'});
+new NC.SRRTable({handler: lhs_handler, selector: '#srr_data'});
+
 {% comment %}</script>{% endcomment %}
 {% endblock graph-code %}