Skip to content
Permalink
Browse files

Merge remote-tracking branch 'upstream/develop' into feature/jean/inf…

…ra-update
  • Loading branch information...
rbreslow committed Dec 5, 2018
2 parents 2a5812d + 8071450 commit ef556c3ce13ea6ebb97598d20d848de24d1e7236
@@ -1,3 +1,4 @@
HOST=localhost
WEB_APP_PORT=8080
WEB_APP_PASSWORD=
ADMIN_USER=admin
@@ -11,6 +11,7 @@ sld
geoserver/data_dir
data
config_settings.py
*-backup.xsd

# Terraform
deployment/terraform/.terraform
@@ -20,4 +21,4 @@ deployment/terraform/.terraform
*.retry

# User data
deployment/user-data
deployment/user-data
@@ -4,10 +4,10 @@ services:
env:
global:
- AWS_DEFAULT_REGION=us-east-1
- secure: W3UWMkZlhIJhnRd7Lulye7f6CfuYMErTe+8K3ekRD25DFE3d4vUXJMeO1G9flkiAEojzG1aqKeO3JLL5c7E+kq9SnojttUBEhlkJYhcSwVAMqxsxcbUzRCsRh+et/y49rVHE52SDYBedC0U2UUnrorJoofeSH/IY4w8j5XJXgQeEO1wGm5wEuxfOxMCecvHb5ZVpxDYrQhVCYk1TPU8IglDn4ybYIzekcPWxnZYZCZLsueFRYSx3esJRVThHDSA1jPX/dsMOpaX1/XbWSeZZ1l+95p6lUgzVtUl4qf4IB08qZyDfs8R3ilu+mdcT5fdoSHZKc7fgveNr6k3aZys8E/uf/1FU0eLOF7vJO8Q1h2S+yRZhM29fHc8C5HJewsGt6EP3sufjg7JKkydXZ2Iy6DVLmQ+4q0ucYWqT4dqufSA06HqIa0v5NPwHeheOfbUXDH73y1rtvy4KfGQHwn1kHYlspg3XLmhO7n65oDc2YwRAMdJ93JUde5ruiIDr/6UwPwc837F+7TeluQO3NIu0e5L6syRH4HQM6nOa14JrUmsZYP2BjmFY9aphuNmGpk6Du5q2xtOnKl+4cQFAXPaL4Ape57R2nnm304g6XLsAr6Y/D7gHeTjdpRofx2mC+6+8nUykBpljGgvbm2fxBCJHEYrvv47owcF6pRdrXG//yq4=
- secure: nI9KxVxRkMYB779SOTtpHn6AeP+GY1fEQlgTs33eyK2wKVME1+jbn6C5pahoVPxI2RUGRf0lw8rjQLBWPkL95me1wHn1nTbVYNsbaenQMyPWANFUg4cFxAnWWpoz1bH37PGC++ahehH7/ebhxgpYNBWLAvmMvJLGCRBDJED515GcnfIL7fW9z5tun0hBUflCZkJh1Iva6SQCGIm6yJwFzUSkHkddE59hlIx2XABSNg8fKzCH0GThHZkKfMDfh+ZLAJuJ26Nvjnkcpy1yfQbc0NJBa0erV3b7GnpB4ewc98Vj/8+iFwloJUhLgRjjVMAZ9M0YMWro6LpazQbXyy4NUkVkZnJuaEGkKZqGU3yYeRSlcZVQyVQGd0LwWV4sqYQ01Yq/YIIyN8EGb4msAwezgzKprZD7yvpRxy/Xgi9TP12aRLPQSeE2YyDeH4RPFGHx2qSYyMbRkdQeRR9LmY3AtlLJhOebwQC35jHz1jNwKkFk4HNb6fgW0p2BsSDUUMEpVwkwuuFcZZTaxKibZ5HIOozKEkXSVOBFIlnRXYFY2QuKrE459PojHU4Jh1bx6RxgSP9TiovF8CM670veZNvzCW2US4258cbFWWJzsaTsxOA0KNqdQXjQJitvl3Z9xLxDoTe0lukj//sp1tybFzfgaLFQQuUw6HG7+W6AcyC7j54=
- secure: Usk91YDPVi6z27MRl1nWscG5rWIWs6g8/C3/tdd/2+g3AQUhPPOX0eNJuqYehRv1U7RNocUQnGXYzgIDmXTEx3uQY4qH8++jPn3AUeTClqKcQrI7gTTXqWHPhauzGokn2x0PU/WtumtRVG0RJGHG19Nb1J+An1ByOacYUTFJuVixMEUlHr7YDpibAKEkjm6+c45yrbfl1TxddmAayNSfmUWbfL+wJNgitv/PqR/gdiv2zHfUgdFoD+sXkojHHp8beqwhMMa8EEGh48/3tcr3TmEV17CSAYOoNi9pUG9TZKnQy0u7qG3leaMOKXdC2BnnTYBRQaAxUWDo3QF5IZeU+6Uobs7zxyvSzYqfrda2aqRMeCGUqkHZ8D0ZdI0Ezn3gKSZcdcuCs80ESyjf6ls2ekJBeFTxfdVjod7b6jhKzxzaikrM8u+4Esdn18ED8eXifRj/OrYHTVV8ysne+nlRBDDds1y07DvZE2qOicl8QOFw6fkAoWEbsVvOFku2sHUtFsOs8ategAcVZTW22g8aX1PmGX8d9nOhWGwIBBCMPMOItBSHb0+dD3HIABYKeGjYrSYV8Xw8OrmNPgg5VkOyLcTszALglvkPjVl5dRvMh86iVPasavYkcnO/wFsQz0vt4/aSGkWuZJ62hfRSwEN6lCF+c2y9Z/VJ5I47n3JoUn0=
- secure: f3YY8IEacet+dShRufP5eA9fe2ulPKZ9mFdDMvqzeMz24K64lT8OIJ11qD4hNVlKL6wxq0uIaxwx6Q8mVDTqqm+SSMNbYvA2FrrwM4xO+S0bd70B0bRdWDUnVyY+dcchV8UnGbC9PoyBUcVi+bWaG+pZN6bG4jnGXB31ynTUJTb1bNh37LOAI2+DqjMGE4h4YYwP669CRl47FUKPJElx4jIOIZkJHxAyZ1LziAZuvsZ195LoV7Uj+rpu2oAaDGRKThXXJ1XrY1dtwOsmmxRk7aumMKIImV5lwve1MCsWi9LPdhoJvgF7dXAGqVl9j0eTgxAL2hELDlhK/7Xci3Cx04N+yaJPFMfoB7yleL2QPqxx88DnsbymNLJPtqbm1IZV7kv5ogy4+vGMOtx5MdpvxjcjlgesKMm3H6CCO+Xwe2g1Ym3UQ7Dt45YHcMpiO0yyH0CJdk49SUioiElNQ4be/F7+CMY+55ISijdmSXaU35Kyy1M46Znn4cfl7Tmiw2xQ7PfzvP4JmieqWWR4Rldc7zZWtCQaSm1BeiiTcFS/SVjgbY10uSo4gFuHm/uPrya4RLI6qC3Oj/1mWiWtPPr/pr7UMHOSMoK4ccGIpA3T5lXg/nOFziwjD/9HMDa25SMwcaPvZkf7wF9HuxcM8U8xYfkdIv01lepe6wX4tDH/9ag=
- secure: IGCVQAaPbhBW47WUGtlOftOpPM7yIz3jk61niBmtF/ACIO9m8Y7KU2+Rl3MwM+4xGv4v0py9kpqAjk7FPCS6PCGA2pUpjMFuXiX7Dfh5Gsy1C0E/j2fvoTkOXkuterw0BcIwi7epymoY4aA8X0GinzuJ1B3sLJXqX8wfH49PSW0=
- secure: HgBnUcWY+L4XCzSBmg1JqCkMeN+5yoA23XTdTbJ0N5K8YXUpzP58FoYkwbhd5sjdnkcSr+2gnGzzxCQ9yxqiD0S78omjYPVurGntwGFdRjy9tATNpWbFqk9sViDX8ge+jx3+DYDIutXvZPZwrS4gkShHrByZrkQJvQTFBjwWkPQ=
- secure: ZtiP7tz+wmIHTzMHoBu6zm0yY02xRAJHwC37/jDbkkmEAxN4WqDGqYRH7acmsfeaWnuiyeBR1pT+yYRfuqKtEVju8s40nHSB1IT5FWHYlFNBRLKxEGKtChEPhtH8C9tfWYfhstal1+paac57tTsIGIRcmTRDwM+Z37qulWWAlYA=
- secure: VuyX3iVEe+vufr4o4+dVooyZ8kk3Vv68g0aGFWysJCuvwlvM1oFwuc4MfybfXmHr433Unxn5DtbHyeeyRuOG9KTdcKlE6ESsxzuiF4RI+CVz8FbjE5tmQXbGsVTkNASYFOLyppLv5d7Wu93Phu7cJYzBOYbcN7winTBRDl9/5KI=
install:
- pip install awscli
- cp .env.sample .env
@@ -32,13 +32,13 @@ deploy:
skip_cleanup: true
script: DB_SETTINGS_BUCKET=${PROD_DB_SETTINGS_BUCKET} DB_DOCKER_HOST=${PROD_DB_DOCKER_HOST} scripts/deploy
on:
repo: azavea/district-builder-dtl-pa
repo: PublicMapping/DistrictBuilder
branch: master
- provider: script
skip_cleanup: true
script: scripts/deploy
on:
repo: azavea/district-builder-dtl-pa
repo: PublicMapping/DistrictBuilder
branch: develop
after_deploy:
- docker-compose -f docker-compose.ci.yml run --rm --entrypoint git terraform clean -fdx
@@ -31,18 +31,16 @@ The development environment is docker-compose for services inside a Vagrant virt
## tl;dr ##

```bash
$ # Copy config and make any necessary edits
$ cp django/publicmapping/config/config.dist.xml django/publicmapping/config/config.xml
$ # Copy .env file and add passwords
$ cp .env.sample .env
$ ./scripts/setup
$ vagrant ssh
$ ./scripts/update
```

If you want to get DistrictBuilder up and running quickly with demo data, you can then run
If you want to get DistrictBuilder up and running quickly with PA data, you can then run
```bash
$ ./scripts/configure_va_demo
$ ./scripts/configure_pa_data
```

Otherwise, you'll need to provide your own shapefiles and config.xml file. Put your zipped shapefile in
@@ -82,12 +80,12 @@ From there, running `./scripts/update` builds containers. The rest of the setup
directly or indirectly through a setup management command. To get started, run
`./scripts/setup`, followed by `vagrant ssh`, followed by `./scripts/update`.

Then, run `./scripts/configure_va_demo`. It is not fast. Currently, it takes several hours, with the exact
Then, run `./scripts/configure_pa_data`. It is not fast. Currently, it takes several hours, with the exact
time depending on hardware. We are working on ways to improve the speed of loading data.

The script will do several things

- Fetch zipped shapefile data for Virginia into a specific location
- Fetch zipped shapefile data for Pennsylvania into a specific location
- Drop and recreate the `district_builder**
- Run database migrations: create the relationships that data will be loaded into
- Load shapes from shapefiles at different levels: create records for the shapes and characteristics
@@ -155,6 +153,7 @@ You can then verify the translations are correct and commit those files.

## Support


More information about the application settings, configuration information, and run-time information is available in the PublicMapping/DistrictBuilder [wiki](https://github.com/PublicMapping/DistrictBuilder/wiki).

Bug reports and feature requests can be reported to the PublicMapping/DistrictBuilder [issue tracker](https://github.com/PublicMapping/DistrictBuilder/issues).
@@ -259,5 +258,3 @@ Finally, start services:
```bash
$ ./scripts/server --production
```


@@ -16,7 +16,8 @@ Vagrant.configure(2) do |config|
"reports/",
"geoserver/data_dir/",
"django/publicmapping/publicmapping/config_settings.py",
"django/publicmapping/locale/"],
"django/publicmapping/locale/**/*.mo",
"django/publicmapping/locale/**/xmlconfig.po"],
rsync__args: ["--verbose", "--archive", "--delete", "-z", "--links"]

config.vm.provider :virtualbox do |vb|
@@ -15,6 +15,7 @@ Amazon Web Services deployment is driven by [Terraform](https://terraform.io/),
* [User Data](#user-data)
* [`scripts/infra`](#scriptsinfra)
* [Loading Shapefile data](#loading-shapefile-data)
* [Fixing Staging](#fixing-staging)

## Deployment

@@ -101,3 +102,15 @@ $ bash recreate_staging_from_scratch
# Remove script
$ rm recreate_staging_from_scratch
```
#### Fixing Pink Tiles
Make sure the `pmp` workspace exists:
1. Log into Geoserver (`http://origin.<domain>:8080/geoserver`). The admin username is `admin` and the password is available as an environment variable (`$MAP_SERVER_ADMIN_PASSWORD`) on the app server within the `districtbuilder-django` container (`docker exec -ti districtbuilder-django bash` and then `echo $MAP_SERVER_ADMIN_PASSWORD`).
1. Create new `pmp` workspace if it doesn't exit (Name = `pmp`, Namespace URI = `https://github.com/PublicMapping/`)
1. Reconfigure geoserver (`./manage.py setup config/config.xml -G` within the `districtbuilder-django` container)
#### Duplicate Records
Reporting being broken and the leaderboard saying "No display configured" is caused by duplicate records in the database. The easiest fix is to drop and recreate the database.
@@ -78,7 +78,7 @@ resource "null_resource" "provision_app_server" {
provisioner "remote-exec" {
inline = [
"sudo mkdir -p /opt/district-builder/user-data",
"sudo mv /home/ec2-user/user-data /opt/district-builder",
"sudo mv /home/ec2-user/user-data/* /opt/district-builder/user-data",
"sudo chown -R ec2-user:ec2-user /opt/district-builder/",
"touch /opt/district-builder/user-data/config_settings.py",
]
@@ -0,0 +1,9 @@
project = "District Builder"
state = "My State"
environment = "Staging"
aws_region = "us-east-1"

app_server_instance_type = "t2.small"
ssl_certificate_arn = "arn:aws:acm:us-east-1:1234567891011:certificate/1a2b3c4d-a5b6-c7d8-e9f1-g2h3i4j5k6l"

cdn_price_class = "PriceClass_100"
@@ -68,4 +68,5 @@ runcmd:
- mkdir -p /etc/docker/certs
- unzip -d /etc/docker/certs /tmp/certs.zip
- echo 'OPTIONS="$${OPTIONS} -H unix:///var/run/docker.sock -H 0.0.0.0:2476 --default-ulimit nofile=1024:4096 --tlsverify --tlscacert=/etc/docker/certs/ca.pem --tlscert=/etc/docker/certs/server.pem --tlskey=/etc/docker/certs/server.key"' >> /etc/sysconfig/docker
- service docker restart
- service docker restart

@@ -213,6 +213,12 @@

USE_TZ = True

ROSETTA_POFILENAMES = (
'django.po',
'djangojs.po',
'xmlconfig.po',
)

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/

@@ -134,6 +134,12 @@ def add_arguments(self, parser):
action='store_true',
help="Create and compile a message file for each language defined.",
default=False)
parser.add_argument(
'-u',
'--updatefield',
dest="updatefield",
action="append",
help="Subject field to update.")

def setup_logging(self, verbosity):
"""
@@ -215,6 +221,9 @@ def handle(self, *args, **options):
geoutil = SpatialUtils(store)

try:
updatefield = None
if options.get("updatefield") is not None:
updatefield = options.get("updatefield")[0]
optlevels = options.get("geolevels")
nestlevels = options.get("nesting")

@@ -226,7 +235,8 @@ def handle(self, *args, **options):
if optlevels is not None:
importme = len(optlevels) == 0 or (i in optlevels)
if importme:
self.import_geolevel(store, geolevel)
self.import_geolevel(
store, geolevel, updatefield)

if nestlevels is not None:
nestme = len(nestlevels) == 0
@@ -298,7 +308,7 @@ def handle(self, *args, **options):
# means that an error occurred - the opposite of the meaning of all_ok's bool
sys.exit(not all_ok)

def import_geolevel(self, store, geolevel):
def import_geolevel(self, store, geolevel, updatefield):
"""
Import the geography at a geolevel.
@@ -340,7 +350,7 @@ def import_geolevel(self, store, geolevel):
sconfig.append(salconfig)
gconfig['subject_fields'].append(sconfig)

self.import_shape(store, gconfig)
self.import_shape(store, gconfig, updatefield)

def import_prereq(self, config, force):
"""
@@ -365,7 +375,7 @@ def import_prereq(self, config, force):

return success

def import_shape(self, store, config):
def import_shape(self, store, config, updatefield):
"""
Import a shapefile, based on a config.
@@ -543,7 +553,8 @@ def get_shape_name(shapefile, feature):
g,
subject_objects,
feat,
updates_possible=not should_create)
not should_create,
updatefield)

logger.info('100%')

@@ -577,24 +588,31 @@ def get_shape_name(shapefile, feature):

if g.count() > 0:
self.set_geounit_characteristic(
g[0], subject_objects, feat)
g[0],
subject_objects,
feat,
True,
updatefield)

logger.info('100%')

def set_geounit_characteristic(self,
g,
subject_objects,
feat,
updates_possible=True):
updates_possible,
updatefield):
to_be_inserted = []
for attr, obj in subject_objects.iteritems():
if attr.endswith('_by_id'):
continue
if updatefield is not None:
if attr != updatefield:
continue
try:
value = Decimal(str(feat.get(attr))).quantize(
Decimal('000000.0000', 'ROUND_DOWN'))
except:

# logger.info('No attribute "%s" on feature %d' , attr, feat.fid)
continue
percentage = '0000.00000000'
@@ -3109,7 +3109,7 @@ def get_contiguity_overrides(self):
objects whose two referenced geounits both fall within
the geometry of this district.
"""
if not self.geom:
if not self.geom or ContiguityOverride.objects.count() == 0:
return []

filter = Q(override_geounit__geom__within=self.geom)
@@ -437,7 +437,6 @@ def commonplan(request, planid):
plan = Plan.objects.filter(id=planid)
if plan.count() == 1:
plan = plan[0]
plan.edited = getutc(plan.edited)
levels = plan.legislative_body.get_geolevels()
districts = plan.get_districts_at_version(
plan.version, include_geom=False)
@@ -955,6 +954,7 @@ def uploadfile(request):
filename = '%s%s' % (dest.name, '.zip')
else:
filename = dest.name
os.chmod(filename, 0664)

except Exception as ex:
logger.error('Could not save uploaded file')
@@ -1180,7 +1180,7 @@ def newdistrict(request, planid):
status['success'] = True
status['message'] = _('Created 1 new district')
plan = Plan.objects.get(pk=planid, owner=request.user)
status['edited'] = getutc(plan.edited).isoformat()
status['edited'] = plan.edited.isoformat()
status['district_id'] = district_id
status['version'] = plan.version
except ValidationError:
@@ -1642,7 +1642,7 @@ def addtodistrict(request, planid, districtid):
% {'num_fixed_districts': fixed}
status['updated'] = fixed
plan = Plan.objects.get(pk=planid, owner=request.user)
status['edited'] = getutc(plan.edited).isoformat()
status['edited'] = plan.edited.isoformat()
status['version'] = plan.version
except Exception, ex:
status['exception'] = traceback.format_exc()
@@ -2547,7 +2547,7 @@ def statistics_sets(request, planid):
admin_displays = ScoreDisplay.objects.filter(
owner__is_superuser=True,
legislative_body=plan.legislative_body,
name__in=admin_display_names)
name__in=admin_display_names).order_by('title')

for admin_display in admin_displays:
sets.append({
@@ -5,6 +5,8 @@ gevent==1.2.2
Django==1.11.10
psycopg2==2.7.3
celery[redis]==4.1.0
redis==2.10.6 # See https://github.com/celery/celery/issues/5175
kombu==4.1.0
django-redis==4.8.0
django-rosetta==0.7.13
django-staticfiles==1.2.1
Oops, something went wrong.

0 comments on commit ef556c3

Please sign in to comment.
You can’t perform that action at this time.