Merge branch 'dev'

conf/pillar/devs.sls

@@ -8,9 +8,6 @@ users:
   djyoung:
     public_key:
       - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDd+HZPZFY4cs7xeuJQXd5XLoeMLpQSLvX7pVOAlXrxiCrk1HxyWfuE6lDy8gjmJffde31SfhIvngwyoA2bQ5ufYT7ronyWHpJGtpDqFL8bFQbtg35Y6VJB7NnIyCFvoQW0GXcXVgipZBJk5bZLfS3BPIoHD1m+1WTFTqDWBKMmn/3l+Yi/cK1iUawlZMOPbPskNzNXuupdOhTf5pzI0ExQGCvISJj7pvMWfq2GMqXKzH5BP7RBn2N2QUB1U1CJfvAcpN/NoF1V+Q1jEGeV03ShayK0eBfc/8HP/7+O/QC/xnUnTHoXlAj7+ECx6YXdTeC8OKI0NCbnG9ArExEdmXxd dyoung@RENCI-W530
-  trawick:
-     public_key:
-      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAdX6zHUvXSdFkqDOlcqWmHb8Gm+McbS1+uLDJzulyGjpbIU/6qRxnIWdmqx75sgMNhMqzPZIsHTh1LBcmeAhodBLy1wX9tn7FgxPqrlprQxZbZCnCFT3zcXoqyGcvH2ZqhQ1lbVPaD5O7yD4RCuOInFFGuQOmVSjoJt58S5KiO2soVYTzxjf46vCxYnmLhscbUd54gJWAWzmqCwuJ3L1FlVi0Vw9KKCXN+0Zav5R3UFcUUQu4YhmBHgDqoxUttY2tKsyTqBo/DSddr0KPysRzFB/i302Xlg1f4vG/VzTeywyS+Jrt3c3EIX1ofbWDHZYGCormfD3LmsZTJRrE589J trawick@trawick-ip
   nmashton:
     public_key:
       - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDN4FV20g1PVrfOakM3LHBrd8Ei6q3TK5XsQNIOHcgVaZuAbYZ6WghUAAeukU+3fYQII2nmZ5FZgfnoBRhPKXa9Qm3yRpEkwzF3W4H40BY2F1fWyfNQuO8C8B4dOQoMbCC53DBGoWdpDYB4FcQ6f404ooAJ4ESM4BOLgMPHJqUygmQZ+Krrwwe8x8+g+guOjgS0Jy4AFYiE1Wr7NA2K2ZVZuSg2k/Lrd4K4173fxbToVxcBjNI+IXRKrWDiz9tpHyP40IltO/TuG08m3N/oTLaM78e1zUHW5iXxaHgzuHNTAkGc+fgzpjaHS5fKQ+YbYUXBxk1SRMKu14v82oXB+hPR nmashton@quaderno.local
@@ -20,3 +17,12 @@ users:
   kmtracey:
     public_key:
       - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEAzGMEy4xHn4kaoXDF1gOxkN4V/L+K4FR0TimE5P2lf89WayMulVejZ12bbkXPRFNxNLQa6sBkrO2GR6CM131cFiXfDB3HR18mAOWOpmvE/XCup/Ksumd2h2wY845Zn7aubbOyUfK87+jW/nXZ1HI+cotnZdbquauCk5/gAhJPg64XDqUSBvW8C9cUCmFifWQE2sWnvWECkDblJCpMhx7zyj6aM1v0gu2fEr74eb+AKtzGuruN81/6ZYgK//gvbTGjVuvKS3mDwMdYlVBnPrzqORtAQzFGc2bAQDEYZWOVWzHW6POWXxYVqYpQIr+yTktRfdnHLcvFGTJsO6nLdXhB6/cXP6w6wumThW4nNdQnQHeWq3obRUDHazGFVYyWUh3QsWVS7tBoDAXxOUynQpX5JfM2ZpKxRnS68UKOGcuVtIFsSSzhWpV4uNEJjfgNfgCutsX5yOYvgUH9rLLSXBrocpIKR1EndySo+vS7oYGDrZFewatApD20uPgWvU/WyPCLHkXHwmVJUqcKPOpasCCGv/WUKaIhF325vA7xVa621k/o9AtJomjyRRE/qVEarqYaJ22xrDJei5M3KM6luo0itVH+HL6jasOqMKa0VSmAX1YIpZX4yQSwEaOxSqlgypWuWrl1LJLN7mD5YnyJBv4gbeN/0YKiWo/TncEdrN+naks= kmtracey@kmt-laptop
+  dchukhin:
+    public_key:
+      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDCyaAGW8HiLvLP9whINKHck0faKwryO++lqA+GYD9G2XZkbsxcBKqU1b3STGn8jXrDjtr4FcKXFGLq8uOS4/FOiCs45TwLOcA4enyTbaPxrZA05Zw5J5hYZ1y4bBvc+GxytAoyySN3MTG7Q2Mb3TgTTl5G/J+d+zGljHBbrjn1Y3xCRPi9ed82bzqdI+6S8kk7kzub++EScjegTfbPhtsyrY4WbZ0Z/hW0Odtt4rXhsktdfLpYTZ3jJs9YapOBFHblRRlZxtc45o5Q2+4B8z8dr1t3jiFqJJdTL5Cnn+DzD/CHiJE/hT8hc4XcHJPjfymMlX2c6AhxN87RcOWHplTGirkNwSnsyquM4fzZPlXpR7aPbTXQNjGrryk95EEkes+/RmdES6QCptGCqGtcmCRFvIYh4v50r7OvyU12GJUmN3HWyDcLMY6JmDvfYzz2ETBeMPWVP++N4IVDzG08WenjA4vTkw2Yvwaodbhe8m0vodqfkNX/ll13wMKf6TAKLFOLc8NfRh4xP936d4xL7pcsGS6LC6pVd6JjJ3CKKHqGcxRuw/00IzFoRRgs+fzIhmXUKbrFtAJbkXaYpV/rfzRc+no3ddWwZ4hK0FaQ5YBU1rfkCBoJzzIWFdJc8dqfbSMwMohHtFZGDhX9seJoo5KZFUbzl0qSD2hJrehGB3k3fQ== dchukhin@gmail.com
+  kialam:
+    public_key:
+      - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC6bBGMXvyLmIHL1H1FJ6AMVbqJfVIqKfGXGPBQ6JIXO5gz6jBtzbXf41Srd4mr5uWn+kRz3w13nIFyCwKIeDQSBivxYcFfiWTiOnEWuTkdNE6mFayXwX/zKcjX9mdC4mjxNT5yDM4Q/i+KG8Ac0fcliSTc2AfWIxjhJoCPBDfuPxVh2VRRvyifIJqdzO0Pv4kcWtnYu2e4hzOQ5nhlJzlBYnABVgpwyaFA3w6PA5g/VDn/F+6Sjo6gWuWtbxAsbfoEuyyuLRzKDuRkufgb4Bq2EZPr0YBNOOWuv6BXuTWgq8Jec4ggK3KGgZl3PEfTxij7J4RGZ1pGE06VEW3GXmKQDVpuyrNFbuHGyXCICR/5S5WWwmM4BCTB6Belhv9xmm7MMmyw5vdH9tQNgB/OTtY3og3ZkkYHrhmYbrgHsPJpuxYdZG1Xm5LTTq7Szl2hjUeOZCBw7OjwM078EHFbYmZzud5T2MezHcHRH5hD/dvPtbmt4Xr9iuoMrC62+7kaZltyqJGzMu5773eP26Ld3hXF47AtTn8D+O0BYGs6aDxmAFgHsnqX80folBzwsaBIkMPYrXMneC/IoKIUvIDL5mlFaTNY86wFHwJXqS6K7lZKKekmusoMQUpEiYVP3zHuDfGv52Rt6agOF77g36KhRs3xTXV29nd2nERrfOB3N+EHrw== digitalanime@gmail.com
+  mlavin:
+    public_key:
+      - ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAm4ppCtMeHOCBTn0gLXRUut31op73cFLFXdQHcD5HWOWaemhqdvFlv7EhWu6fdNHuusRt1j592y+Mswk1l5cmt2jv/1TwSgIoJ2YNUNv0tDWAuMAT5wcE8lj/PEN6294M9WQyRWQW6r8ZRCz78m//rkZZeDvu0xiT6ZJtaldZqhq3UbHssjxpNBp1ooT80D1AuT7c5y5neJSxaQ1JNMOuihheY8rtSfz3U2/0MnurMZXdvO6X+2hVszrmp0VNrvFHsiPoH5dOmmGjhXi8DbNpX1pWhYQinaREEslSiUwQbdE6drQ30uR+6dqon2WULWE3KPUtYVDn+DoKKxi1UJXYrw== mlavin@caktus007

conf/pillar/production.sls

@@ -17,7 +17,7 @@ repo:
   branch: master
 
 postgresql_config: # from pgtune
-  work_mem: 88MB
+  work_mem: 200MB
   maintenance_work_mem: 448MB
   shared_buffers: 1792MB
   effective_cache_size: 5GB
@@ -27,8 +27,8 @@ postgresql_config: # from pgtune
 
 # Addtional public environment variables to set for the project
 env:
-  FOO: BAR
-
+    NEW_RELIC_APP_NAME: "opendatapolicing production"
+    NEW_RELIC_MONITOR_MODE: "true"
 
 # Uncomment and update username/password to enable HTTP basic auth
 # Password must be GPG encrypted.

conf/pillar/staging.sls

@@ -11,12 +11,13 @@ repo:
   branch: dev
 
 postgresql_config: # from pgtune
-  work_mem: 44MB
+  work_mem: 200MB
   maintenance_work_mem: 448MB
   shared_buffers: 1792MB
   effective_cache_size: 5GB
   checkpoint_segments: 32
   log_min_duration_statement: 1000
+  random_page_cost: 1
 
 # Addtional public environment variables to set for the project
 env:

conf/salt/project/web/gunicorn.conf

@@ -1,5 +1,5 @@
 [program:{{ pillar['project_name'] }}-server]
-command={{ directory }}/dotenv.sh {% if use_newrelic %}{{ virtualenv_root }}/bin/newrelic-admin run-program {% endif %}{{ virtualenv_root }}/bin/gunicorn {{ pillar['project_name'] }}.wsgi:application --bind=0.0.0.0:8000 --workers={{ grains['num_cpus'] * 2 + 1 }} --timeout=300
+command={{ directory }}/dotenv.sh {% if use_newrelic %}{{ virtualenv_root }}/bin/newrelic-admin run-program {% endif %}{{ virtualenv_root }}/bin/gunicorn {{ pillar['project_name'] }}.wsgi:application --bind=0.0.0.0:8000 --workers={{ grains['num_cpus'] * 2 + 1 }} --timeout=500
 user={{ pillar['project_name'] }}
 directory={{ directory }}
 autostart=true

docs/data-import.rst

@@ -9,9 +9,16 @@ environments, using the ``import_census`` management command.  This must be
 performed as part of developer and server setup as well as when census support is
 added for additional states.
 
+
 Local/Development Environment
 -----------------------------
 
+Before running state imports, first import census data:
+
+.. code-block:: bash
+
+  python manage.py import_census
+
 
 Database Dump (quicker)
 _______________________
@@ -22,9 +29,9 @@ To load an existing database dump on S3, run:
 
     dropdb traffic_stops_nc
     createdb -E UTF-8 traffic_stops_nc
-    wget https://s3-us-west-2.amazonaws.com/openpolicingdata/traffic_stops_nc_production_20150801.tar.zip
-    unzip traffic_stops_nc_production_20150801.tar.zip
-    pg_restore -Ox -d traffic_stops_nc traffic_stops_nc_production.tar
+    wget https://s3-us-west-2.amazonaws.com/openpolicingdata/traffic_stops_nc_2018_01_08.dump.zip
+    unzip traffic_stops_nc_2018_01_08.dump.zip
+    pg_restore -Ox -d traffic_stops_nc traffic_stops_nc_2018_01_08.dump
 
 Browse https://s3-us-west-2.amazonaws.com/openpolicingdata/ to see what dumps
 are available.
@@ -33,10 +40,11 @@ To create a new database dump, run:
 
 .. code-block:: bash
 
-    pg_dump -Ox -Ft traffic_stops > traffic_stops.tar
+    ssh dev.opendatapolicingnc.com 'sudo -u postgres pg_dump -Fc traffic_stops_nc_staging' > traffic_stops_nc.dump
 
 That can be loaded with the ``pg_restore`` command shown above.
 
+
 Raw NC Data (slower)
 ____________________
 
@@ -45,17 +53,35 @@ applied before importing.  If in doubt:
 
 .. code-block:: bash
 
+    # for NC
     dropdb traffic_stops_nc && createdb -E UTF-8 traffic_stops_nc
+    # for MD
+    dropdb traffic_stops_md && createdb -E UTF-8 traffic_stops_md
+
     ./migrate_all_dbs.sh
 
+
 Command-line
 ++++++++++++
 
+If loading NC, make sure to add ``NC_FTP_USER`` and ``NC_FTP_PASSWORD`` and your ``.env`` file.
+
+If on a Mac, install ``gnu-sed``:
+
+.. code-block:: bash
+
+  brew install gnu-sed --with-default-names
+
 Run the import command:
 
 .. code-block:: bash
 
-    python manage.py import_nc --dest $PWD/ncdata
+    # for NC (~25m)
+    rm -rf ./ncdata  # if you don't want to reuse previous download
+    python manage.py import_nc --dest $PWD/ncdata --noprime  # noprime = don't prime cache
+    # for MD (~30m)
+    rm -rf ./mddata  # if you don't want to reuse previous download
+    python manage.py import_md --dest $PWD/mddata
 
 This took ~25 minutes on my laptop. Run ``tail -f traffic_stops.log`` to follow
 along.  Reusing an existing ``--dest`` directory will speed up import.  However,
@@ -68,6 +94,7 @@ Now you should be able to view data with ``runserver``:
 
     python manage.py runserver
 
+
 Admin
 +++++
 
@@ -90,6 +117,7 @@ imported.  Setting the fields:
 Once the "dataset" has been created, select the new dataset in list view and
 apply the "Import selected dataset" action.
 
+
 Server
 ------
 
@@ -123,9 +151,11 @@ activated prior to an import of IL data and then deactivated afterwards, as foll
     sudo swapoff /swapfile
     sudo rm /swapfile
 
+
 Raw NC Data
 ___________
 
+
 Command-line
 ++++++++++++
 
@@ -142,11 +172,13 @@ Reusing an existing ``--dest`` directory will speed up import.  However,
 if import code has changed since the last time the directory was used, don't
 reuse an existing directory.
 
+
 Admin
 +++++
 
 Follow the "Admin" instructions above under "Local/Development Environment".
 
+
 Create DB Dump
 ______________
 
@@ -157,6 +189,7 @@ ______________
     # then on local laptop, run:
     scp opendatapolicingnc.com:traffic_stops_nc_production.tar.zip .
 
+
 Updating landing page stats
 ---------------------------
 
@@ -196,4 +229,3 @@ management commands.  Example::
     Id 224: Raleigh Police Department 863,653
     Id 104: Greensboro Police Department 555,453
     Id 88: Fayetteville Police Department 503,013
-

docs/frontend.rst

@@ -37,14 +37,6 @@ Each subclass of ``VisualBase`` needs to define at least these methods:
 - ``setDefaultChart``: setting the ``this.chart`` value to an NVD3 chart with correct configuration values
 - ``drawChart``: calling ``this.chart`` to actually mount and draw the chart with the data
 
-There is also an optional ``triggerRaceToggle`` method that is triggered by the ``raceToggle.change`` event on the ``document``. This takes care of switching out the graph's data between "race" and "ethnicity" views, where this is relevant. For example, within a NC chart, this method is defined like so::
-
-    triggerRaceToggle: function (e, v) {
-      this.set('showEthnicity', v);
-      this.drawChart();
-    }
-
-From this example, you can see that a chart that allows different data sets to be selected will need to have an event handler that invokes ``this.drawChart`` in order to redraw the chart after the needed local state updates have been performed.
 
 Understanding "common" classes
 ------------------------------
@@ -59,14 +51,14 @@ Let's look at ``StopRatioDonutBase`` as a concrete example. This has two abstrac
 The North Carolina instance of this class defines them like this::
 
     _items: function () {
-      return (this.get('showEthnicity')) ? Stops.ethnicities : Stops.races;
+      return Stops.ethnicities;
     },
 
     _pprint: function (type) {
       return Stops.pprint.get(type);
     }
 
-So when ``_items`` is invoked, it fetches its ``showEthnicity`` value and returns either ``Stops.ethnicities`` or ``Stops.races``. When ``_pprint`` is used, it looks up a value in ``Stops.pprint``.
+So when ``_items`` is invoked, it fetches ``Stops.ethnicities``. When ``_pprint`` is used, it looks up a value in ``Stops.pprint``.
 
 The Maryland instance is much simpler::
 
@@ -78,7 +70,7 @@ The Maryland instance is much simpler::
       return x;
     }
 
-Because Maryland has no race / ethnicity distinction, its items are simply ``Stops.ethnicities``. And because its data is "humanized" from the get-go, its ``_pprint`` function can just return the input value.
+Because its data is "humanized" from the get-go, its ``_pprint`` function can just return the input value.
 
 Other configuration
 -------------------