Skip to content

Commit

Permalink
Adapt search API for loading status
Browse files Browse the repository at this point in the history
  • Loading branch information
@akariv
akariv committed Sep 28, 2016
1 parent 7ca9bed commit 89951ee
Show file tree
Hide file tree
Showing 5 changed files with 92 additions and 17 deletions.
8 changes: 5 additions & 3 deletions conductor/blueprints/package/controllers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,17 @@ def toggle_publish(name, token, toggle=False, publish=False):
return None

name, datapackage_url, datapackage, \
model, dataset_name, author = package_registry.get_raw(name)
model, dataset_name, author,\
status, loaded = package_registry.get_raw(name)
private = datapackage.get('private', False)
if toggle:
private = not private
else:
private = not publish
datapackage['private'] = private
package_registry.save_model(name, datapackage_url, datapackage,
model, dataset_name, author)
model, dataset_name, author,
status, loaded)
return {'success': True, 'published': not private}

obeu_url = 'http://eis-openbudgets.iais.fraunhofer.de/' \
Expand All @@ -110,7 +112,7 @@ def run_hooks(name, token):
except jwt.InvalidTokenError:
return None

_, datapackage_url, _, _, _, _ = package_registry.get_raw(name)
_, datapackage_url, _, _, _, _, _, _ = package_registry.get_raw(name)
json_ld_payload = {
"@context": {
"@vocab": "http://schema.org/",
Expand Down
28 changes: 16 additions & 12 deletions conductor/blueprints/search/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
'package': {
'index': 'packages',
'doc_type': 'package',
'_source': ['id', 'model', 'package', 'origin_url'],
'_source': ['id', 'model', 'package', 'origin_url', 'loaded'],
'owner': 'package.owner',
'private': 'package.private',
'q_fields': ['package.title',
Expand All @@ -40,31 +40,35 @@ def _get_engine():


def build_dsl(kind_params, userid, kw):
dsl = {'bool': {'must': []}}
privacy = ({
dsl = {'bool': {'should': [], 'must': [], 'minimum_should_match': 1}}
# All Datasets:
all_datasets = {
'bool': {
'should': [{'match': {kind_params['private']: False}},
{'filtered':
{'filter': {'missing': {'field':
kind_params['private']}}}},
]
],
'must_not': {'match': {'loaded': False}}
}
})
}
dsl['bool']['should'].append(all_datasets)

# User datasets
if userid is not None:
privacy['bool']['should']\
.append({'match': {kind_params['owner']: userid}})
user_datasets = \
{'bool': {'must': {'match': {kind_params['owner']: userid}}}}
dsl['bool']['should'].append(user_datasets)

dsl['bool']['must'].append(privacy)
# Query parameters
q = kw.get('q')
if q is not None:
dsl['bool']['must'].append(
{
dsl['bool']['must'].append({
'multi_match': {
'query': json.loads(q[0]),
'fields': kind_params['q_fields']
}
}
)
})
for k, v_arr in kw.items():
if k.split('.')[0] in kind_params['_source']:
dsl['bool']['must'].append({
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,4 @@ sqlalchemy
psycopg2
cryptography
elasticsearch>=1.0.0,<2.0.0
os-package-registry>=0.0.2
os-package-registry>=0.0.3
2 changes: 1 addition & 1 deletion tests/module/blueprints/package/test_controllers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ def setUp(self):
time.sleep(1)

self.pr = PackageRegistry(es_connection_string=LOCAL_ELASTICSEARCH)
self.pr.save_model('name','datapackage_url',{},{},'dataset','author')
self.pr.save_model('name','datapackage_url',{},{},'dataset','author','',True)

def test__initial_value__none(self):
pkg = self.pr.get_package('name')
Expand Down
69 changes: 69 additions & 0 deletions tests/module/blueprints/search/test_controllers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,26 @@ def indexSomeRealLookingRecords(self, amount):
self.es.index('packages', 'package', body)
self.es.indices.flush('packages')

def indexSomePrivateRecords(self):
i = 0
for owner in ['owner1', 'owner2']:
for private in [True, False]:
for loaded in [True, False]:
for content in ['cat', 'dog']:
body = {
'id': '%s-%s-%s-%s' % (owner, private, loaded, content),
'package': {
'author': 'The one and only author number%d' % (i+1),
'title': 'This dataset is number%d, content is %s' % (i, content),
'owner': owner,
'private': private
},
'loaded': loaded
}
self.es.index('packages', 'package', body)
i += 1
self.es.indices.flush('packages')

# Tests
def test___search___all_values_and_empty(self):
self.assertEquals(len(module.search('package', None)), 0)
Expand Down Expand Up @@ -106,3 +126,52 @@ def test___search___q_param_some_recs_no_results(self):
def test___search___q_param_some_recs_some_results(self):
self.indexSomeRealLookingRecords(2)
self.assertEquals(len(module.search('package', None, {'q': ['"number1"']})), 2)

def test___search___empty_anonymous_search(self):
self.indexSomePrivateRecords()
recs = module.search('package', None)
self.assertEquals(len(recs), 4)
ids = set([r['id'] for r in recs])
self.assertSetEqual(ids, {'owner1-False-True-cat',
'owner2-False-True-cat',
'owner1-False-True-dog',
'owner2-False-True-dog',
})

def test___search___empty_authenticated_search(self):
self.indexSomePrivateRecords()
recs = module.search('package', 'owner1')
ids = set([r['id'] for r in recs])
self.assertSetEqual(ids, {'owner1-False-False-cat',
'owner1-False-True-cat',
'owner1-True-False-cat',
'owner1-True-True-cat',
'owner2-False-True-cat',
'owner1-False-False-dog',
'owner1-False-True-dog',
'owner1-True-False-dog',
'owner1-True-True-dog',
'owner2-False-True-dog',
})
self.assertEquals(len(recs), 10)

def test___search___q_param_anonymous_search(self):
self.indexSomePrivateRecords()
recs = module.search('package', None, {'q': ['"cat"']})
self.assertEquals(len(recs), 2)
ids = set([r['id'] for r in recs])
self.assertSetEqual(ids, {'owner1-False-True-cat',
'owner2-False-True-cat',
})

def test___search___q_param_authenticated_search(self):
self.indexSomePrivateRecords()
recs = module.search('package', 'owner1', {'q': ['"cat"']})
ids = set([r['id'] for r in recs])
self.assertSetEqual(ids, {'owner1-False-False-cat',
'owner1-False-True-cat',
'owner1-True-False-cat',
'owner1-True-True-cat',
'owner2-False-True-cat',
})
self.assertEquals(len(recs), 5)

0 comments on commit 89951ee

Please sign in to comment.