diff --git a/README.md b/README.md index 5aa8450..aff38f5 100755 --- a/README.md +++ b/README.md @@ -31,21 +31,21 @@ Feel free to use these examples as a jumping off point in your own code. ### Requirements - Docker 17+ (or Discovery Binaries) -- Python 2.7 with `pip` +- Python 3 with `pip` ### Dependencies To ensure your dependencies install correctly, we recommend that you upgrade your `setuptools` before proceeding further. ```sh -pip install --upgrade setuptools +python3 -m pip install --upgrade setuptools ``` Now you are ready to install the required dependencies with `pip`. This will provide you with the packages needed to run the `test_discovery.py` script, as well as the Discovery CLI. ```sh -pip install -r requirements.txt +python3 -m pip install -r requirements.txt ``` ### Obtaining credentials @@ -73,7 +73,7 @@ Ensure that your *transcripts are unformatted text with numbers spelled out*. Fo 5) Edit the credentials and other configuration parameters in `discovery_config.py` -6) Run `python test_discovery.py folder_name` to test your interpreter. For example: +6) Run `python3 test_discovery.py folder_name` to test your interpreter. For example: ``` python test_discovery.py examples/directions @@ -90,7 +90,7 @@ You should end up with a structure like the following for the `test_discovery.py ``` └───greenkey-discovery-sdk - └───discovery_binaries_windows_10_64bit__python27_32bit + └───discovery_binaries_windows_10_64bit__python37_64bit └───examples └───gkcli │ discovery_config.py @@ -131,7 +131,7 @@ Then, run these files through SVTServer [following our documentation](https://tr For example, you can launch a single job container with the following command for a file called `test.wav`. Be sure to set `$USERNAME` and `$SECRETKEY` -``` +```bash docker run \ -it \ --rm \ @@ -140,7 +140,7 @@ docker run \ -e GKT_USERNAME="$USERNAME" \ -e GKT_SECRETKEY="$SECRETKEY" \ -e TARGET_FILE="/files/test.wav" \ - -v $(pwd):/files \ + -v "$PWD":/files \ -e ENABLE_CLOUD="False" \ -e PROCS=1 \ docker.greenkeytech.com/svtserver diff --git a/discovery_sdk_utils/discovery_sdk_utils/built_in_tokens.py b/discovery_sdk_utils/discovery_sdk_utils/built_in_tokens.py index dc92e70..1be8930 100755 --- a/discovery_sdk_utils/discovery_sdk_utils/built_in_tokens.py +++ b/discovery_sdk_utils/discovery_sdk_utils/built_in_tokens.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 ONTOLOGICAL_TAGS = ( 'IN', diff --git a/discovery_sdk_utils/discovery_sdk_utils/entity_validation_funcs.py b/discovery_sdk_utils/discovery_sdk_utils/entity_validation_funcs.py index 215f6a0..727af10 100755 --- a/discovery_sdk_utils/discovery_sdk_utils/entity_validation_funcs.py +++ b/discovery_sdk_utils/discovery_sdk_utils/entity_validation_funcs.py @@ -1,7 +1,7 @@ -#!/usr/bin/python -''' +#!/usr/bin/env python3 +""" Utility functions for discovery -''' +""" from __future__ import print_function from inspect import isfunction from .built_in_tokens import BUILT_IN_TOKENS @@ -221,8 +221,7 @@ def _type_is_correct(item, type): def find_errors_in_entity_definition(entity_definition, validation_functions=VALIDATION_FUNCTIONS): - ''' Accumulates a list of errors to return to the user. - ''' + """Accumulates a list of errors to return to the user.""" errors = [] for func in validation_functions: # Minor errors are returned as strings and added to the list. diff --git a/examples/directions/custom/entities/direction.py b/examples/directions/custom/entities/direction.py index e13e3b8..4cd4cc8 100755 --- a/examples/directions/custom/entities/direction.py +++ b/examples/directions/custom/entities/direction.py @@ -1,5 +1,9 @@ ''' This contains the entity definition for a cardinal directions ''' -from cleaning_functions import capitalize +try: + from cleaning_functions import capitalize +except ImportError: + from .cleaning_functions import capitalize + DIRECTION = { 'label': 'DIRECTION', diff --git a/examples/directions/custom/entities/street_name.py b/examples/directions/custom/entities/street_name.py index eb2e003..99349c4 100755 --- a/examples/directions/custom/entities/street_name.py +++ b/examples/directions/custom/entities/street_name.py @@ -2,9 +2,11 @@ This contains the entity definition for the most popular street names in Chicago, Illinois. Replace these streets with whatever street names you please. ''' -import sys -sys.path.append('../../nlp/') -from cleaning_functions import capitalize +from nlp.cleanText import text2int +try: + from cleaning_functions import capitalize +except ImportError: + from .cleaning_functions import capitalize STREET_NAME = { 'label': @@ -134,7 +136,6 @@ def format_ordinal(wordList, spacer): >>> format_ordinal("five three", ' ') '5th 3rd' """ - from cleanText import text2int def add_suffix(n): return str(n) + 'tsnrhtdd' [n % 5 * (n % 100 ^ 15 > 4 > n % 10)::4] diff --git a/examples/directions/custom/entities/street_type.py b/examples/directions/custom/entities/street_type.py index 5709e8a..9b848a3 100755 --- a/examples/directions/custom/entities/street_type.py +++ b/examples/directions/custom/entities/street_type.py @@ -1,8 +1,10 @@ ''' This contains the entity definition for types of streets. ''' - -from cleaning_functions import capitalize +try: + from cleaning_functions import capitalize +except ImportError: + from .cleaning_functions import capitalize STREET_TYPE = { 'label': diff --git a/examples/directions/send_transcript_to_discovery.sh b/examples/directions/send_transcript_to_discovery.sh index 44c4766..cfae0fa 100755 --- a/examples/directions/send_transcript_to_discovery.sh +++ b/examples/directions/send_transcript_to_discovery.sh @@ -1,5 +1,5 @@ #!/bin/bash curl -X POST http://localhost:1234/discover \ - -H "Content-type: multipart/form-data" \ - -F 'data={"json_lattice": {"transcript": "I need directions to fifty five west monroe avenue by car"}};type=application/json' + -H "Content-Type: application/json" \ + -d '{"transcript": "I need directions to fifty five west monroe avenue by car"}' diff --git a/examples/room_dialing/send_transcript_to_discovery.sh b/examples/room_dialing/send_transcript_to_discovery.sh index 51d999e..631b65b 100755 --- a/examples/room_dialing/send_transcript_to_discovery.sh +++ b/examples/room_dialing/send_transcript_to_discovery.sh @@ -1,5 +1,5 @@ #!/bin/bash curl -X POST http://localhost:1234/discover \ - -H "Content-type: multipart/form-data" \ - -F 'data={"json_lattice": {"transcript": "dial one eight"}};type=application/json' + -H "Content-Type: application/json" \ + -d '{"transcript": "dial one eight"}' diff --git a/examples/room_number/send_transcript_to_discovery.sh b/examples/room_number/send_transcript_to_discovery.sh index 5089c01..70b5631 100755 --- a/examples/room_number/send_transcript_to_discovery.sh +++ b/examples/room_number/send_transcript_to_discovery.sh @@ -1,5 +1,5 @@ #!/bin/bash curl -X POST http://localhost:1234/discover \ - -H "Content-type: multipart/form-data" \ - -F 'data={"json_lattice": {"transcript": "five a is calling thirteen c"}};type=application/json' + -H "Content-Type: application/json" \ + -d '{"transcript": "five a is calling thirteen c"}' diff --git a/examples/telephone_number/send_transcript_to_discovery.sh b/examples/telephone_number/send_transcript_to_discovery.sh index e739ccf..d825f12 100755 --- a/examples/telephone_number/send_transcript_to_discovery.sh +++ b/examples/telephone_number/send_transcript_to_discovery.sh @@ -1,5 +1,5 @@ #!/bin/bash curl -X POST http://localhost:1234/discover \ - -H "Content-type: multipart/form-data" \ - -F 'data={"json_lattice": {"transcript": "You can reach me at five five five four three two five nine eight one"}};type=application/json' + -H "Content-Type: application/json" \ + -d '{"transcript": "You can reach me at five five five four three two five nine eight one"}' diff --git a/launch_discovery.py b/launch_discovery.py index bc80205..5381089 100755 --- a/launch_discovery.py +++ b/launch_discovery.py @@ -1,25 +1,28 @@ #!/usr/bin/env python -'''This module will launch Discovery on http://localhost:1234 +"""This module will launch Discovery on http://localhost:1234 This file will automatically scan for a folder of discovery binaries. If the binaries are not detected, the script will launch the Discovery docker container. Available functions: - launch_discovery: Starts discovery container or binaries -''' +""" import fnmatch +from multiprocessing import Process import os import subprocess +import sys + from discovery_config import DISCOVERY_CONFIG, DISCOVERY_PORT, DISCOVERY_IMAGE_NAME def launch_discovery(custom_directory=None, type=None, port=None, discovery_config=None): - '''Launches the Discovery engine either via docker container or via compiled binaries. + """Launches the Discovery engine either via docker container or via compiled binaries. Args: custom_directory: File path to the custom directory would like uploaded to discovery. type: String, can either be 'docker' or 'binaries' - ''' + """ if custom_directory is None: custom_directory = os.getcwd() if port is None: @@ -43,7 +46,7 @@ def _determine_discovery_launch_type(): def _launch_container(custom_directory, port, discovery_config): - '''launches the Discovery docker container.''' + """launches the Discovery docker container.""" dico_dir = ["-v", "{}/dico:/dico".format(custom_directory)] if os.path.isdir(custom_directory + '/dico') else [] try: @@ -68,10 +71,9 @@ def _launch_container(custom_directory, port, discovery_config): def _launch_binaries(custom_directory, port, discovery_config): - '''launches Discovery from the compiled binaries.''' + """launches Discovery from the compiled binaries.""" binaries_directory = _detect_binaries_file() if binaries_directory: - import sys sys.path.append(os.path.abspath(binaries_directory)) from run import find_definition_files_and_copy_them_to_appropriate_location find_definition_files_and_copy_them_to_appropriate_location( @@ -79,20 +81,18 @@ def _launch_binaries(custom_directory, port, discovery_config): ) sys.path.append(os.path.join(binaries_directory, 'discovery')) - os.chdir(os.path.join(binaries_directory, 'discovery', 'server')) + print("Launching Discovery from Binaries: {}\n".format(binaries_directory)) - sdk_directory = os.path.abspath(os.path.join(binaries_directory, '..')) - # remove any instances of the SDK directory from the path to prevent dummy modules from getting loaded in - while sdk_directory in sys.path: - sys.path.remove(sdk_directory) + os.environ["SERVICE_NAME"] = "discovery" + from discovery.server import main - from server.server import main - main() + proc = Process(target=main, args=()) + proc.start() def _detect_binaries_file(): - '''Returns the absolute path of the binaries directory.''' + """Returns the absolute path of the binaries directory.""" for file in os.listdir('.'): if fnmatch.fnmatch(file, 'discovery_binaries_*') and not file.endswith('.tar.gz') and not file.endswith('.zip'): return os.path.abspath(file) diff --git a/test_discovery.py b/test_discovery.py index e376ec5..4e0dab1 100755 --- a/test_discovery.py +++ b/test_discovery.py @@ -1,5 +1,5 @@ -#!/usr/bin/python -''' +#!/usr/bin/env python3 +""" Test Discovery performs system testing of new intents and interpreters. This tool is intended to be distributed to any developer who wishes to develop their own @@ -8,7 +8,7 @@ Tests are assumed to pass if the defined entities are present in the most likely found intent. Tests are also assumed to always return a valid intent with entities. -''' +""" from __future__ import print_function import requests @@ -31,15 +31,15 @@ DISCOVERY_DIRECTORY = os.getcwd() TEST_FILE = os.path.join(DISCOVERY_DIRECTORY, "tests.txt") -''' +""" Functions for handling the Discovery Docker container -''' +""" def check_discovery_status(): - ''' + """ Checks whether Discovery is ready to receive new jobs - ''' + """ r = requests.get("http://{}:{}/status".format(DISCOVERY_HOST, DISCOVERY_PORT)) if json.loads(r.text)['status'] == 0: @@ -49,9 +49,9 @@ def check_discovery_status(): def wait_for_discovery_status(timeout=1, retries=5): - ''' + """ Wait for Discovery to be ready - ''' + """ for i in range(retries): try: check_discovery_status() @@ -63,9 +63,9 @@ def wait_for_discovery_status(timeout=1, retries=5): def wait_for_discovery_launch(): - ''' + """ Wait for launch to complete - ''' + """ # Timeout of 25 seconds for launch if not wait_for_discovery_status(timeout=5, retries=5): @@ -76,9 +76,9 @@ def wait_for_discovery_launch(): def shutdown_discovery(): - ''' + """ Shuts down the Discovery engine Docker container - ''' + """ try: requests.get("http://{}:{}/shutdown".format(DISCOVERY_HOST, DISCOVERY_PORT)) # Windows throws a ConnectionError for a request to shutdown a server which makes it looks like the test fail @@ -86,15 +86,15 @@ def shutdown_discovery(): pass -''' +""" Testing functions -''' +""" def load_tests(): - ''' + """ Loads and parses the test file - ''' + """ test_file = [x.rstrip() for x in open(TEST_FILE)] tests = [] @@ -116,20 +116,20 @@ def load_tests(): def submit_transcript(transcript): - ''' + """ Submits a transcript to Discovery - ''' - d = {"data": json.dumps({"json_lattice": {"transcript": transcript}})} - r = requests.post("http://{}:{}/discover".format(DISCOVERY_HOST, DISCOVERY_PORT), data=d) + """ + data = {"transcript": transcript} + response = requests.post("http://{}:{}/process".format(DISCOVERY_HOST, DISCOVERY_PORT), json=data) - return json.loads(r.text) + return json.loads(response.text) def is_valid_response(resp): - ''' + """ Validates a Discovery response Fail if a failure response was received - ''' + """ if "result" in resp and resp['result'] == "failure": return False @@ -146,9 +146,9 @@ def is_valid_response(resp): def test_single_entity(entities, test_name, test_value): - ''' + """ Tests a single entity within a test case - ''' + """ if test_name not in entities.keys(): fail_test({}, "Entity not found: {}".format(test_name), continued=True) @@ -166,10 +166,10 @@ def test_single_entity(entities, test_name, test_value): def test_single_case(test): - ''' + """ Run a single test case Return the number of errors - ''' + """ print("======\nTesting: {}".format(test['test'])) resp = submit_transcript(test['transcript']) @@ -190,7 +190,7 @@ def test_single_case(test): # Loop through all entity tests for test_name, test_value in test.items(): if test_name in ['test', 'transcript']: - continue + continue (errors, char_errors) = test_single_entity(entities, test_name, test_value) total_errors += errors @@ -210,9 +210,9 @@ def test_single_case(test): def test_all(): - ''' + """ Runs all defined tests - ''' + """ tests = load_tests() t1 = int(time.time()) @@ -249,15 +249,25 @@ def fail_test(resp, message="", continued=False): exit(1) -''' +""" Interpreter validation -''' +""" + + +class cleanText(object): + """Mock up a module that is imported by entities so they can be imported and inspected.""" + @staticmethod + def text2int(wordList, spacer): + return wordList def validate_entities(): - ''' + """ Validate entities - ''' + """ + # mock up nlp.cleanText so that entities can be imported + sys.modules['nlp.cleanText'] = cleanText() + entities_file = os.path.join(DISCOVERY_DIRECTORY, 'custom', 'entities') entities = glob.glob(os.path.join(entities_file, '*.py')) entities_directory = os.path.join(DISCOVERY_DIRECTORY, 'custom', 'entities') @@ -272,18 +282,21 @@ def validate_entities(): except FileNotFoundError: pass + # remove mocked up dummy module from modules + sys.modules.pop('nlp.cleanText') + def _validate_individual_entity(entity): - entity_name = os.path.split(entity)[-1].replace(".py", "") - entity_module = import_module(entity_name) - definition_errors = [] - if 'ENTITY_DEFINITION' in dir(entity_module): - print('Checking entity definition {0:.<35}'.format(entity_name), end='') - errors = find_errors_in_entity_definition(entity_module.ENTITY_DEFINITION) - _log_entity_definition_error_results(errors) - definition_errors.extend(errors) - if definition_errors: - raise Exception('Please fix all entity definition errors before running discovery!') + entity_name = os.path.split(entity)[-1].replace(".py", "") + entity_module = import_module(entity_name) + definition_errors = [] + if 'ENTITY_DEFINITION' in dir(entity_module): + print('Checking entity definition {0:.<35}'.format(entity_name), end='') + errors = find_errors_in_entity_definition(entity_module.ENTITY_DEFINITION) + _log_entity_definition_error_results(errors) + definition_errors.extend(errors) + if definition_errors: + raise Exception('Please fix all entity definition errors before running discovery!') def _log_entity_definition_error_results(errors): @@ -296,9 +309,9 @@ def _log_entity_definition_error_results(errors): def validate_json(): - ''' + """ Validate intents.json - ''' + """ intents_config_file = os.path.join(DISCOVERY_DIRECTORY, 'custom', 'intents.json') try: json.loads(''.join([x.rstrip() for x in open(intents_config_file)]))