-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use separate /function API endpoints, add UDF tests, extract server m…
…ock into separate module
- Loading branch information
1 parent
1661fab
commit 7dae515
Showing
8 changed files
with
221 additions
and
94 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,3 @@ | ||
| [run] | ||
| source = myria | ||
| omit = myria/utility/cloudpickle.py |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| import json | ||
| from datetime import datetime | ||
|
|
||
| from httmock import urlmatch | ||
| from myria.udf import MyriaPythonFunction | ||
| from raco.types import STRING_TYPE, LONG_TYPE | ||
|
|
||
| RELATION_NAME = 'relation' | ||
| FULL_NAME = 'public:adhoc:' + RELATION_NAME | ||
| QUALIFIED_NAME = {'userName': 'public', | ||
| 'programName': 'adhoc', | ||
| 'relationName': RELATION_NAME} | ||
| NAME_COMPONENTS = ['public', 'adhoc', RELATION_NAME] | ||
| SCHEMA = {'columnNames': ['column', 'column2'], | ||
| 'columnTypes': ['INT_TYPE', 'INT_TYPE']} | ||
| CREATED_DATE = datetime(1900, 1, 2, 3, 4) | ||
| TUPLES = [[1, 9], [2, 8], [3, 7], [4, 6], [5, 5]] | ||
| TOTAL_TUPLES = len(TUPLES) | ||
|
|
||
| RELATION_NAME2 = 'relation2' | ||
| FULL_NAME2 = 'public:adhoc:' + RELATION_NAME2 | ||
| QUALIFIED_NAME2 = {'userName': 'public', | ||
| 'programName': 'adhoc', | ||
| 'relationName': RELATION_NAME2} | ||
| NAME_COMPONENTS2 = ['public', 'adhoc', RELATION_NAME2] | ||
| SCHEMA2 = {'columnNames': ['column3', 'column4'], | ||
| 'columnTypes': ['INT_TYPE', 'INT_TYPE']} | ||
| TUPLES2 = [[1, 9], [2, 8], [3, 7], [4, 6], [5, 5]] | ||
| TOTAL_TUPLES2 = len(TUPLES2) | ||
|
|
||
| UDF1_NAME, UDF2_NAME = 'udf1', 'udf2' | ||
| UDF1_TYPE, UDF2_TYPE = LONG_TYPE, STRING_TYPE | ||
| UDF1_ARITY, UDF2_ARITY = 1, 2 | ||
|
|
||
|
|
||
| def get_uri(name): | ||
| return '/dataset/user-{}/program-{}/relation-{}'.format( | ||
| 'public', 'adhoc', name) | ||
|
|
||
|
|
||
| def create_mock(state=None): | ||
| state = state if state is not None else {} | ||
|
|
||
| @urlmatch(netloc=r'localhost:12345') | ||
| def local_mock(url, request): | ||
| # Relation metadata | ||
| if url.path == get_uri(RELATION_NAME): | ||
| body = {'numTuples': TOTAL_TUPLES, | ||
| 'schema': SCHEMA, | ||
| 'created': str(CREATED_DATE)} | ||
| return {'status_code': 200, 'content': body} | ||
| elif url.path == get_uri(RELATION_NAME2): | ||
| body = {'numTuples': TOTAL_TUPLES2, | ||
| 'schema': SCHEMA2, | ||
| 'created': str(CREATED_DATE)} | ||
| return {'status_code': 200, 'content': body} | ||
|
|
||
| # Relation download | ||
| if url.path == get_uri(RELATION_NAME) + '/data': | ||
| body = str(TUPLES) | ||
| return {'status_code': 200, 'content': body} | ||
| elif url.path == get_uri(RELATION_NAME2) + '/data': | ||
| body = str(TUPLES2) | ||
| return {'status_code': 200, 'content': body} | ||
|
|
||
| # Relation not found in database | ||
| elif get_uri('NOTFOUND') in url.path: | ||
| return {'status_code': 404} | ||
|
|
||
| elif url.path == '/function' and request.method == 'GET': | ||
| body = [UDF1_NAME, UDF2_NAME] | ||
| return {'status_code': 200, 'content': body} | ||
|
|
||
| elif url.path == '/function/' + UDF1_NAME and request.method == 'GET': | ||
| return { | ||
| 'status_code': 200, | ||
| 'content': MyriaPythonFunction( | ||
| lambda i: 0, UDF1_TYPE, UDF1_NAME, False).to_dict()} | ||
|
|
||
| elif url.path == '/function/' + UDF2_NAME and request.method == 'GET': | ||
| return { | ||
| 'status_code': 200, | ||
| 'content': MyriaPythonFunction( | ||
| lambda i: 0, UDF2_TYPE, UDF2_NAME, False).to_dict()} | ||
|
|
||
| elif url.path == '/function' and request.method == 'POST': | ||
| body = json.loads(request.body) | ||
| state[body['name']] = body | ||
| return {'status_code': 200, 'content': '{}'} | ||
|
|
||
| return None | ||
| return local_mock |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| import unittest | ||
|
|
||
| from httmock import HTTMock | ||
| from myria.connection import MyriaConnection | ||
| from myria.test.mock import * | ||
| from myria.udf import MyriaFunction, MyriaPostgresFunction, myria_function | ||
| from raco.backends.myria.connection import FunctionTypes | ||
| from raco.myrial.parser import Parser | ||
|
|
||
|
|
||
| class TestUDF(unittest.TestCase): | ||
| def __init__(self, args): | ||
| with HTTMock(create_mock()): | ||
| self.connection = MyriaConnection(hostname='localhost', port=12345) | ||
| super(TestUDF, self).__init__(args) | ||
|
|
||
| def test_get_all(self): | ||
| with HTTMock(create_mock()): | ||
| functions = MyriaFunction.get_all(self.connection) | ||
|
|
||
| self.assertGreaterEqual(len(functions), 2) | ||
|
|
||
| self.assertEqual(functions[0].language, FunctionTypes.PYTHON) | ||
| self.assertEqual(functions[1].language, FunctionTypes.PYTHON) | ||
|
|
||
| self.assertEqual(functions[0].name, UDF1_NAME) | ||
| self.assertEqual(functions[1].name, UDF2_NAME) | ||
|
|
||
| self.assertEqual(functions[0].output_type, UDF1_TYPE) | ||
| self.assertEqual(functions[1].output_type, UDF2_TYPE) | ||
|
|
||
| def test_get(self): | ||
| with HTTMock(create_mock()): | ||
| function = MyriaFunction.get(UDF1_NAME, self.connection) | ||
|
|
||
| self.assertEqual(function.language, FunctionTypes.PYTHON) | ||
| self.assertEqual(function.name, UDF1_NAME) | ||
| self.assertEqual(function.output_type, UDF1_TYPE) | ||
|
|
||
| def test_register(self): | ||
| server_state = {} | ||
| with HTTMock(create_mock(server_state)): | ||
| f = MyriaFunction('mockudf', 'source', STRING_TYPE, | ||
| FunctionTypes.PYTHON, False, | ||
| connection=self.connection) | ||
| f.register() | ||
|
|
||
| self.assertEqual(len(server_state), 1) | ||
| self.assertDictEqual(f.to_dict(), server_state.values()[0]) | ||
| self.assertFalse(server_state.values()[0]['isMultiValued']) | ||
| self.assertEqual(server_state.values()[0]['outputType'], | ||
| 'STRING_TYPE') | ||
|
|
||
| def test_python_udf(self): | ||
| name = 'pyudf' | ||
| server_state = {} | ||
|
|
||
| with HTTMock(create_mock(server_state)): | ||
| f = MyriaPythonFunction(lambda: False, STRING_TYPE, name, | ||
| multivalued=False, | ||
| connection=self.connection) | ||
| f.register() | ||
| d = MyriaPythonFunction.from_dict(server_state[name]).to_dict() | ||
|
|
||
| self.assertEqual(len(server_state), 1) | ||
| self.assertIsNotNone(MyriaFunction.get(name, self.connection)) | ||
| self.assertIn(name, Parser.udf_functions) | ||
|
|
||
| self.assertEqual(f.to_dict()['name'], d['name']) | ||
| self.assertEqual(f.to_dict()['outputType'], d['outputType']) | ||
| self.assertEqual(f.to_dict()['lang'], d['lang']) | ||
| self.assertEqual(f.to_dict()['isMultiValued'], d['isMultiValued']) | ||
|
|
||
| def test_postgres_udf(self): | ||
| name = 'postgresudf' | ||
| server_state = {} | ||
|
|
||
| with HTTMock(create_mock(server_state)): | ||
| f = MyriaPostgresFunction(name, 'source', STRING_TYPE, | ||
| multivalued=False, | ||
| connection=self.connection) | ||
| f.register() | ||
| d = MyriaPythonFunction.from_dict(server_state[name]).to_dict() | ||
|
|
||
| self.assertEqual(len(server_state), 1) | ||
| self.assertIsNotNone(MyriaFunction.get(name, self.connection)) | ||
| self.assertIn(name, Parser.udf_functions) | ||
|
|
||
| self.assertEqual(f.to_dict()['name'], d['name']) | ||
| self.assertEqual(f.to_dict()['outputType'], d['outputType']) | ||
| self.assertEqual(f.to_dict()['isMultiValued'], d['isMultiValued']) | ||
|
|
||
| def test_extension_method(self): | ||
| server_state = {} | ||
|
|
||
| with HTTMock(create_mock(server_state)): | ||
| name = 'my_udf' | ||
|
|
||
| @myria_function(name=name, output_type=STRING_TYPE, | ||
| connection=self.connection) | ||
| def my_udf(t): | ||
| return None | ||
|
|
||
| self.assertEqual(len(server_state), 1) | ||
| self.assertIsNotNone(MyriaFunction.get(name, self.connection)) | ||
| self.assertIn(name, Parser.udf_functions) | ||
|
|
||
| d = MyriaPythonFunction.from_dict(server_state[name]).to_dict() | ||
| self.assertEqual(d['name'], name) | ||
| self.assertEqual(d['outputType'], STRING_TYPE) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters