Remove Elastic Search (#192)

ElasticSearch is a great tool for use by frontend applications. However,
ElasticSearch (without purchase) does not come with good policy based
attributes access methods. As a result Pacifica must re-architect to
synchronize the metadata into ElasticSearch through policy
administration scripts.

Signed-off-by: David Brown <dmlb2000@gmail.com>

.travis.yml

@@ -31,7 +31,7 @@ env:
   unzip data.zip; pushd pacifica-metadata-0.3.1;
   curl localhost:8121/users;
   python tests/test_files/loadit_test.py; wait $METADATA_PID;
-  popd; popd;
+  popd; popd; sudo service elasticsearch stop;
 - cd tests
 - pip install ..
 - >
@@ -52,7 +52,7 @@ env:
 - coverage run --include='*/site-packages/pacifica/metadata/*' -a core/cmd_test.py dbsync;
 - coverage run --include='*/site-packages/pacifica/metadata/*' -a core/cmd_test.py dbsync;
 - coverage run --include='*/site-packages/pacifica/metadata/*' -a core/cmd_test.py dbchk;
-- coverage run --include='*/site-packages/pacifica/metadata/*' -a -m pytest -xv orm elastic core
+- coverage run --include='*/site-packages/pacifica/metadata/*' -a -m pytest -xv orm core
 - coverage run --include='*/site-packages/pacifica/metadata/*' -a -m pytest -xv rest
 - coverage report -m --fail-under 100
 ".before_script": &2

Dockerfile

@@ -3,7 +3,8 @@ FROM python:3.6
 WORKDIR /usr/src/app
 COPY requirements.txt ./
 RUN pip install --no-cache-dir -r requirements.txt
-RUN pip install --no-cache-dir uwsgi
+RUN pip install --no-cache-dir uwsgi pymysql psycopg2 cryptography
 COPY . .
+RUN pip install .
 EXPOSE 8121
 ENTRYPOINT [ "/bin/bash", "/usr/src/app/entrypoint.sh" ]

appveyor.yml

@@ -18,9 +18,8 @@ environment:
 install:
 - ps: >
     & "$env:PYTHON\python.exe" -m virtualenv C:\pacifica;
-    Invoke-WebRequest -Uri "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.5.zip" -OutFile "elasticsearch.zip";
+    Invoke-WebRequest -Uri "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.14.zip" -OutFile "elasticsearch.zip";
     Expand-Archive "elasticsearch.zip" -DestinationPath "C:\elasticsearch";
-    Start-Process C:\elasticsearch\elasticsearch-5.6.5\bin\elasticsearch;
     C:\pacifica\Scripts\activate.ps1;
     python -m pip install --upgrade pip setuptools wheel;
     python -m pip install -r requirements-dev.txt;
@@ -41,6 +40,7 @@ before_test:
 test_script:
 - ps: >
     mkdir C:\tmp; C:\pacifica\Scripts\activate.ps1;
+    $es_proc = Start-Process C:\elasticsearch\elasticsearch-5.6.14\bin\elasticsearch -PassThru;
     pre-commit run -a;
     pip install pacifica-metadata==0.3.1;
     $env:METADATA_CPCONFIG = "$PWD/server.conf";
@@ -53,7 +53,7 @@ test_script:
     Invoke-WebRequest http://localhost:8121/users;
     python tests\test_files\loadit_test.py;
     $meta_proc | Wait-Process;
-    popd;
+    popd; $es_proc | Stop-Process;
     cd tests;
     pip install ..;
     coverage run --include='*/site-packages/pacifica/metadata/*' core/cmd_test.py dbchk --equal;
@@ -69,6 +69,6 @@ test_script:
     coverage run --include='*/site-packages/pacifica/metadata/*' -a core/cmd_test.py dbsync;
     coverage run --include='*/site-packages/pacifica/metadata/*' -a core/cmd_test.py dbsync;
     coverage run --include='*/site-packages/pacifica/metadata/*' -a core/cmd_test.py dbchk;
-    coverage run --include='*/site-packages/pacifica/metadata/*' -a -m pytest -xv orm elastic core;
+    coverage run --include='*/site-packages/pacifica/metadata/*' -a -m pytest -xv orm core;
     coverage run --include='*/site-packages/pacifica/metadata/*' -a -m pytest -xv rest;
     coverage report -m --fail-under=100;

docker-compose.yml

@@ -1,73 +1,21 @@
 version: '2'
 
 services:
-  elasticmaster:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10
-    environment:
-    - bootstrap.memory_lock=true
-    - xpack.security.enabled=false
-    - xpack.monitoring.enabled=false
-    - xpack.watcher.enabled=false
-    - http.cors.enabled=true
-    - "http.cors.allow-origin=\"/.*/\""
-    - "http.cors.allow-methods=OPTIONS, HEAD, GET, POST, PUT, DELETE"
-    - "http.cors.allow-headers=\"X-Requested-With,X-Auth-Token,Content-Type, Content-Length, Authorization\""
-    - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    mem_limit: 1g
-    ports:
-      - 9200:9200
-
-  elasticslave:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.10
-    environment:
-    - bootstrap.memory_lock=true
-    - xpack.security.enabled=false
-    - xpack.monitoring.enabled=false
-    - xpack.watcher.enabled=false
-    - http.cors.enabled=true
-    - "http.cors.allow-origin=\"/.*/\""
-    - "http.cors.allow-methods=OPTIONS, HEAD, GET, POST, PUT, DELETE"
-    - "http.cors.allow-headers=\"X-Requested-With,X-Auth-Token,Content-Type, Content-Length, Authorization\""
-    - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    - discovery.zen.ping.unicast.hosts=elasticmaster
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    mem_limit: 1g
-
-  elasticlb:
-    image: dockercloud/haproxy
-    links:
-      - elasticslave:elasticslave
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-
   metadatadb:
     image: postgres
-    container_name: metadatadb
     environment:
       POSTGRES_PASSWORD: metadata
       POSTGRES_DB: pacifica_metadata
       POSTGRES_USER: pacifica
-    ports:
-      - 5432:5432
 
   metadataserver:
     build:
-        context: .
-    container_name: metadataserver
+      context: .
     links:
       - metadatadb:postgres
-      - elasticmaster
     ports:
       - 8121:8121
     environment:
-      ELASTICDB_PORT: tcp://elasticmaster:9200
       PEEWEE_DATABASE: pacifica_metadata
       PEEWEE_USER: pacifica
       PEEWEE_ADDR: postgres

entrypoint.sh

@@ -11,10 +11,11 @@ if [[ -z $PEEWEE_DATABASE_URL ]] ; then
   PEEWEE_DATABASE_URL="${PEEWEE_PROTO}://${PEEWEE_USER_PART}${PEEWEE_ADDR_PART}/${PEEWEE_DATABASE}"
 fi
 mkdir ~/.pacifica-metadata/
+cp /usr/src/app/server.conf ~/.pacifica-metadata/cpconfig.ini
 printf '[database]\npeewee_url = '${PEEWEE_DATABASE_URL}'\n' > ~/.pacifica-metadata/config.ini
-python -c 'from pacifica.metadata.admin_cmd import dbsync; dbsync()'
+pacifica-metadata-cmd dbsync
 uwsgi \
   --http-socket 0.0.0.0:8121 \
   --master \
   --die-on-term \
-  --wsgi-file /usr/src/app/pacifica/metadata/wsgi.py "$@"
+  --module pacifica.metadata.wsgi "$@"

pacifica/metadata/admin_cmd.py

@@ -9,8 +9,6 @@
 from peewee import PeeweeException
 from .orm.all_objects import ORM_OBJECTS
 from .orm.sync import OrmSync, MetadataSystem
-from .elastic import try_es_connect, create_elastic_index
-from .essync import escreate, essync
 
 
 def render_obj(args):
@@ -28,14 +26,11 @@ def render_obj(args):
     )
     if args.delete:
         test_obj.delete_instance()
-        test_obj.elastic_delete(test_obj)
 
 
 def create_obj(args):
     """Create a specific object."""
     OrmSync.dbconn_blocking()
-    try_es_connect()
-    create_elastic_index()
     if not args.object.table_exists():
         args.object.create_table()
 
@@ -66,8 +61,6 @@ def dbsync(_args=None):
         MetadataSystem.get_version()
     except PeeweeException:
         OrmSync.dbconn_blocking()
-        try_es_connect()
-        create_elastic_index()
         OrmSync.create_tables()
         OrmSync.close()
         return
@@ -88,16 +81,6 @@ def create_subcommands(subparsers):
         help='render help',
         description='render and object from database w/o API'
     )
-    escreate_parser = subparsers.add_parser(
-        'escreate',
-        help='escreate help',
-        description='create elastic index and mappings'
-    )
-    essync_parser = subparsers.add_parser(
-        'essync',
-        help='essync help',
-        description='sync sql data to elastic search'
-    )
     db_parser = subparsers.add_parser(
         'dbsync',
         help='dbsync help',
@@ -110,8 +93,6 @@ def create_subcommands(subparsers):
     )
     return (
         (render_parser, render_options),
-        (escreate_parser, escreate_options),
-        (essync_parser, essync_options),
         (create_obj_parser, create_obj_options),
         (db_parser, db_options),
         (dbchk_parser, dbchk_options)
@@ -132,43 +113,6 @@ def dbchk_options(dbchk_parser):
     dbchk_parser.set_defaults(func=dbchk)
 
 
-def escreate_options(escreate_parser):
-    """Add the escreate command line options."""
-    escreate_parser.add_argument(
-        '--skip-mappings',
-        help='skip creating mappings',
-        default=False,
-        action='store_true',
-        dest='skip_mappings',
-        required=False
-    )
-    escreate_parser.set_defaults(func=escreate)
-
-
-def essync_options(essync_parser):
-    """Add the essync command line options."""
-    essync_parser.add_argument(
-        '--objects-per-page', default=40000,
-        type=int, help='objects per bulk upload.',
-        required=False, dest='items_per_page'
-    )
-    essync_parser.add_argument(
-        '--threads', default=4, required=False,
-        type=int, help='number of threads to sync data',
-    )
-    essync_parser.add_argument(
-        '--object-name', dest='objects', type=objstr_to_ormobj,
-        help='object type to sync.', required=False, nargs='*',
-        default=ORM_OBJECTS
-    )
-    essync_parser.add_argument(
-        '--time-ago', dest='time_ago', type=objstr_to_timedelta,
-        help='only objects newer than X days ago.', required=False,
-        default=timedelta(days=36500)
-    )
-    essync_parser.set_defaults(func=essync)
-
-
 def objstr_to_timedelta(obj_str):
     """Turn an object string of the format X unit ago into timedelta."""
     value, unit, check = obj_str.split()

pacifica/metadata/config.py

@@ -32,9 +32,6 @@ def get_config():
         'NOTIFICATIONS_DISABLED', 'False'))
     configparser.set('notifications', 'url', getenv(
         'NOTIFICATIONS_URL', 'http://127.0.0.1:8070/receive'))
-    configparser.add_section('elasticsearch')
-    configparser.set('elasticsearch', 'url', getenv(
-        'ELASTIC_ENDPOINT', 'http://127.0.0.1:9200'))
     configparser.read(CONFIG_FILE)
     return configparser