Merge 0e83ad0 into 5e29df9

.ebextensions/.gitignore

@@ -0,0 +1,5 @@
+
+# Elastic Beanstalk Files
+.elasticbeanstalk/*
+!.elasticbeanstalk/*.cfg.yml
+!.elasticbeanstalk/*.global.yml

.ebextensions/01_packages.config

@@ -0,0 +1,28 @@
+packages:
+   yum:
+     git: []
+     python36-virtualenv: []
+     gcc: []
+     gcc-c++: []
+     postgresql96-devel: []     
+     httpd24-devel: []
+files:
+  "/tmp/update-wsgi.sh" :
+    mode: "000755"
+    owner: root
+    group: root
+    content: |
+      # update mod_wsgi
+      cd /tmp
+      wget -q "https://github.com/GrahamDumpleton/mod_wsgi/archive/4.4.21.tar.gz" && \
+      tar -xzf '4.4.21.tar.gz' && \
+      cd ./mod_wsgi-4.4.21 && \
+      sudo ./configure --with-python=/usr/bin/python3.6 && \
+      sudo make && \
+      sudo make install && \
+      sudo service httpd restart
+
+commands:
+  mod_wsgi_update:
+    command: /tmp/update-wsgi.sh
+    cwd: /tmp

.ebextensions/02_python.config

@@ -0,0 +1,31 @@
+---
+option_settings:
+  - namespace: aws:elasticbeanstalk:application:environment
+    option_name: DJANGO_SETTINGS_MODULE
+    value: hamlet.settings.aws
+  - option_name: PYTHONPATH
+    value: "/opt/python/current/app/hamlet:$PYTHONPATH"
+  - option_name: S3_MODELS_BUCKET
+    value: "s3://hamlet-models/"
+  - option_name: MODELS_DIR
+    value: /models
+  - namespace: aws:elasticbeanstalk:container:python
+    option_name: WSGIPath
+    value: hamlet/wsgi.py
+  - namespace: aws:elasticbeanstalk:container:python:staticfiles
+    option_name: /static/
+    value: staticfiles/
+
+container_commands:
+    01_create_models_dir:
+        command: "sudo mkdir --mode=755 ${MODELS_DIR}"
+        test: '[ ! -d "${MODELS_DIR}" ]'
+    02_sync_models_stored_in_s3:
+        command: "aws s3 sync ${S3_MODELS_BUCKET} ${MODELS_DIR}"
+    03_migrate:
+        command: "source /opt/python/run/venv/bin/activate && python manage.py migrate --noinput"
+        leader_only: true
+    04_collectstatic:
+        command: "source /opt/python/run/venv/bin/activate && python manage.py collectstatic --noinput"
+    05_compress:
+        command: "source /opt/python/run/venv/bin/activate && python manage.py compress"

.ebextensions/03_apache.config

@@ -0,0 +1,16 @@
+container_commands:
+  01_setup_apache:
+    command: "cp .ebextensions/enable_mod_deflate.conf /etc/httpd/conf.d/enable_mod_deflate.conf"
+  02_setup_cache_headers:
+    command: "cp .ebextensions/enable_cache_headers.conf /etc/httpd/conf.d/enable_cache_headers.conf"
+
+files:
+    "/etc/httpd/conf.d/ssl_rewrite.conf":
+        mode: "000644"
+        owner: root
+        group: root
+        content: |
+            RewriteEngine On
+            <If "-n '%{HTTP:X-Forwarded-Proto}' && %{HTTP:X-Forwarded-Proto} != 'https'">
+            RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} [R,L]
+            </If>

.ebextensions/04_logging.config

@@ -0,0 +1,5 @@
+commands:
+   01_change_permissions:
+      command: chmod g+s /opt/python/log
+   02_change_owner:
+      command: chown root:wsgi /opt/python/log

.ebextensions/05_elb.config

@@ -0,0 +1,6 @@
+option_settings:
+  aws:elb:listener:443:
+    SSLCertificateId: arn:aws:acm:us-east-1:672626379771:certificate/4d482670-0299-47f3-8250-ca232c5d1bd2
+    ListenerProtocol: HTTPS
+    InstancePort: 80
+

.ebextensions/enable_cache_headers.conf

@@ -0,0 +1,6 @@
+<Directory /opt/python/current/app/staticfiles/>
+    <FilesMatch ".+\.([abcdef0123456789]{12})\.([a-z0-9A-Z]{2,6})$">
+        Header set Cache-Control "max-age=31536000, public"
+    </FilesMatch>
+</Directory>
+

.ebextensions/enable_mod_deflate.conf

@@ -0,0 +1,26 @@
+# mod_deflate configuration
+<IfModule mod_deflate.c>
+  # Restrict compression to these MIME types
+  AddOutputFilterByType DEFLATE text/plain
+  AddOutputFilterByType DEFLATE text/html
+  AddOutputFilterByType DEFLATE application/xhtml+xml
+  AddOutputFilterByType DEFLATE text/xml
+  AddOutputFilterByType DEFLATE application/xml
+  AddOutputFilterByType DEFLATE application/xml+rss
+  AddOutputFilterByType DEFLATE application/x-javascript
+  AddOutputFilterByType DEFLATE text/javascript
+  AddOutputFilterByType DEFLATE text/css
+  # Level of compression (Highest 9 - Lowest 1)
+  DeflateCompressionLevel 9
+  # Netscape 4.x has some problems.
+  BrowserMatch ^Mozilla/4 gzip-only-text/html
+  # Netscape 4.06-4.08 have some more problems
+  BrowserMatch ^Mozilla/4\.0[678] no-gzip
+  # MSIE masquerades as Netscape, but it is fine
+  BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html
+<IfModule mod_headers.c>
+  # Make sure proxies don't deliver the wrong content
+  Header append Vary User-Agent env=!dont-vary
+</IfModule>
+</IfModule>
+

.elasticbeanstalk/config.yml

@@ -0,0 +1,20 @@
+branch-defaults:
+  beanstalk:
+    environment: Hamtest
+environment-defaults:
+  Hamtest:
+    branch: null
+    repository: null
+global:
+  application_name: hamlet
+  default_ec2_keyname: mit-dornera
+  default_platform: arn:aws:elasticbeanstalk:us-east-1::platform/Python 3.6 running
+    on 64bit Amazon Linux/2.6.1
+  default_region: us-east-1
+  include_git_submodules: true
+  instance_profile: null
+  platform_name: null
+  platform_version: null
+  profile: null
+  sc: git
+  workspace_type: Application

.travis.yml

@@ -1,20 +1,30 @@
 language: python
 python:
-  - "3.6"
+- '3.6'
 install:
-  - "pip install pipenv"
-  - "pip install coveralls"
-  - "pipenv install"
-script:
-  coverage run manage.py test
-after_success:
-  coveralls
+- pip install pipenv
+- pip install coveralls
+- pipenv install
+script: coverage run manage.py test
+after_success: coveralls
 services:
-  - postgresql
+- postgresql
 before_script:
-  - psql -c 'create database travis_ci_test;' -U postgres
+- psql -c 'create database travis_ci_test;' -U postgres
 env:
   global:
-    - DJANGO_DB=travis_ci_test
-    - DJANGO_DB_USER=postgres
-    - DJANGO_SETTINGS_MODULE=hamlet.settings.test
+  - DJANGO_DB=travis_ci_test
+  - DJANGO_DB_USER=postgres
+  - DJANGO_SETTINGS_MODULE=hamlet.settings.test
+deploy:
+  provider: elasticbeanstalk
+  access_key_id: AKIAJQXIO6JJ6FWT7HFQ
+  secret_access_key:
+    secure: JFotdLDjfTiRFlhmc1eWIT35kbocYizPgVu96bGZX1gvRudRY6x7SLsha2RPqv80ImkhWkfxhdxnCtvTUoprghk6YhA8Z28pz8BmdiZDT3i/AfGvtETSK5h7Z3Vm6wY3M37cz740+eL9rThlcElTGtz0KNtvnQglPiQJIJqaeTpMFVpV6DDYqiQd+sHo09Lyr709HsNTLq6m+AmlIrnG0keywZjQvwzg+9c73UJ1Z1iEgY0h6fAzHA4C83q1f/NUP1UBk0PFzjHNp0fcDv+Jyjl9BMx5bmz/ug7c6mlpRbTjKrS8ncvlQnR+wXSy1naFYhWJEAxRVmLqviY1PST1URFFOJoTR8bHPOWCV4jib7MFHDb0HW9U1n+6hYYRrJgvAdu9wSeShoRXBNGlX1PiL/xWbZOJFyWw6rLCCUimKci3gCfnGP/VVTlatnOm8P8J8lg6Oy4MypHfJzzhStdspj4BKfv1hQWKQbF1DRseInDjeiE29/wZ/osV7B37ArgBN2o2ikCB4kZnSlvex6zSSHZborgpSRLz36XUqhOaUcd0XbgFu5YVdyTzRjgUV0zFZKtFUqPv49kb9HboSv1D6gQV1gXtWyR8M+GL0B9mZ08om7nj0SMkbZSTNAa+PqyXwmbsYobSQa1llstCe3zuh5V5Sh4luz4uRfeAeqz2nAs=
+  region: us-east-1
+  bucket_name: elasticbeanstalk-us-east-1-672626379771
+  app: hamlet
+  env: hamtest
+  on:
+    repo: MITLibraries/hamlet
+    branch: master

docs/docs.md

@@ -51,6 +51,13 @@ We tried to deploy on Heroku but the model file needs ~2GB of memory and that ge
 
 https://mitlibraries-hamlet.mit.edu/
 
+Environmnet Varibles defined in AWS for security reasons:
+-All Database variables (these are standard and can put directly in your code)
+-SECRET_KEY - will be created by TS3 or provided by developer securely
+
+All other variables are defined with the config files of the .ebextensions folder and can be changed/modified and or added to for future use.
+
+
 ### Deployment
 
 The goal is to have master autodeploy via Travis. Right now if you want something to be deployed, ask Andy.

hamlet/settings/aws.py

@@ -0,0 +1,122 @@
+import os
+import sys
+
+from .base import *  # noqa
+from gensim.models.doc2vec import Doc2Vec
+
+# CHECK FOR EC2 TO ADD IP TO ALLOWED HOSTS
+#-----------------------------------------------------------------------------
+def is_ec2_linux():
+    """Detect if we are running on an EC2 Linux Instance
+        See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
+    """
+    if os.path.isfile("/sys/hypervisor/uuid"):
+        with open("/sys/hypervisor/uuid") as f:
+            uuid = f.read()
+            return uuid.startswith("ec2")
+    return False
+
+def get_linux_ec2_private_ip():
+    """Get the private IP Address of the machine if running on an EC2 linux server"""
+    try:
+        from urllib2 import urlopen
+    except:
+        from urllib.request import urlopen
+    if not is_ec2_linux():
+        return None
+    try:
+        response = urlopen('http://169.254.169.254/latest/meta-data/local-ipv4')
+        return response.read()
+    except:
+        return None
+    finally:
+        if response:
+           response.close()
+#--------------------------------------------------------------------------
+
+
+# DATABASE CONFIGURATION
+# -----------------------------------------------------------------------------
+
+DATABASES = {  
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': os.environ['RDS_DB_NAME'],
+        'USER': os.environ['RDS_USERNAME'],
+        'PASSWORD': os.environ['RDS_PASSWORD'],
+        'HOST': os.environ['RDS_HOSTNAME'],
+        'PORT': os.environ['RDS_PORT'],
+    }
+}
+
+# GENERAL CONFIGURATION
+# -----------------------------------------------------------------------------
+#GET SECRET KEY FROM ENV VARIABLE
+SECRET_KEY = os.environ ['SECRET_KEY']
+
+# Honor the 'X-Forwarded-Proto' header for request.is_secure()
+SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
+
+
+ALLOWED_HOSTS = [ 
+    '.compute-1.amazonaws.com', # allows viewing of instances directly
+    'mitlibraries-hamlet.mit.edu',
+    'localhost',
+ ]
+
+private_ip = get_linux_ec2_private_ip()
+if private_ip:
+   ALLOWED_HOSTS.append(private_ip)
+
+# STATIC FILE CONFIGURATION
+# -----------------------------------------------------------------------------
+
+MIDDLEWARE.insert(1,'whitenoise.middleware.WhiteNoiseMiddleware')
+
+STATIC_URL = '/static/'
+STATIC_ROOT =  os.path.join(BASE_DIR, 'staticfiles')
+
+STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
+
+COMPRESS_ENABLED = True
+COMPRESS_OFFLINE = True
+
+#MODEL FILES STORED ON S3
+#MODELS_DIR IS AN ENV VARIABLE DEFINED IN EB AS /models
+MODELS_DIR = os.environ.get('MODELS_DIR')
+MODEL_FILE = os.path.join(MODELS_DIR, 'hamlet.model')
+NEURAL_NET = Doc2Vec.load(MODEL_FILE)
+
+# LOGGING CONFIGURATION
+# -----------------------------------------------------------------------------
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'formatters': {
+        'brief': {
+            'format': '%(asctime)s %(levelname)s %(name)s[%(funcName)s]: %(message)s',  # noqa
+        },
+    },
+    'handlers': {
+        'console_info': {
+            'level': 'INFO',
+            'class': 'logging.StreamHandler',
+            'stream': sys.stdout
+        },
+    },
+    'loggers': {
+        '': {
+            'handlers': ['console_info'],
+            'level': 'INFO',
+        }
+    }
+}
+
+# Will be emailed by the management command about API usage.
+ADMINS = [('Andromeda Yelton', 'm31@mit.edu')]