Skip to content

Commit

Permalink
documents: store allowed ips addresses in CIDR format
Browse files Browse the repository at this point in the history
Note: this requires a data migration, the document index should be
      recreated.

* Fixes slow OAI-PHM requests that returns 500 errors.
* Fixes slow document api requests.

Co-Authored-by: Johnny Mariéthoz <Johnny.Mariethoz@rero.ch>
  • Loading branch information
jma committed Feb 9, 2022
1 parent cb7fad2 commit 1dece3b
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,9 @@
"type": "text"
},
"ips": {
"type": "ip_range"
},
"allowedIps": {
"type": "keyword"
}
}
Expand Down
5 changes: 3 additions & 2 deletions sonar/modules/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,8 @@ def get_ips_list(ranges):
:param list ranges: List of ranges.
:returns: List of IP addresses.
:rtype: list
:rtype: list of cidr ips
(https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing)
"""
ip_set = IPSet()

Expand All @@ -315,7 +316,7 @@ def get_ips_list(ranges):
except Exception:
pass

return [str(ip) for ip in ip_set]
return [str(ip.cidr) for ip in ip_set.iter_cidrs()]


def file_download_ui(pid, record, _record_file_factory=None, **kwargs):
Expand Down
30 changes: 30 additions & 0 deletions tests/api/documents/test_documents_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,26 @@ def test_masked_document(db, client, organisation, document, es_clear):
assert res.status_code == 200
assert res.json['hits']['total']['value'] == 0

# Masked for external IPs, IP is allowed
organisation['allowedIps'] = '127.0.0.1/32'
organisation.commit()
db.session.commit()
organisation.reindex()
document.reindex()
res = client.get(url_for('invenio_records_rest.doc_list', view='global'))
assert res.status_code == 200
assert res.json['hits']['total']['value'] == 1

# Masked for external IPs, IP is allowed
organisation['allowedIps'] = '127.0.0.*'
organisation.commit()
db.session.commit()
organisation.reindex()
document.reindex()
res = client.get(url_for('invenio_records_rest.doc_list', view='global'))
assert res.status_code == 200
assert res.json['hits']['total']['value'] == 1

# Masked for external IPs, IP is allowed
organisation['allowedIps'] = '127.0.0.1'
organisation.commit()
Expand All @@ -80,3 +100,13 @@ def test_masked_document(db, client, organisation, document, es_clear):
res = client.get(url_for('invenio_records_rest.doc_list', view='global'))
assert res.status_code == 200
assert res.json['hits']['total']['value'] == 1

# Masked for external IPs, IP is not allowed
organisation['allowedIps'] = '192.168.1.1'
organisation.commit()
db.session.commit()
organisation.reindex()
document.reindex()
res = client.get(url_for('invenio_records_rest.doc_list', view='global'))
assert res.status_code == 200
assert res.json['hits']['total']['value'] == 0
6 changes: 3 additions & 3 deletions tests/ui/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,6 @@ def test_get_current_ip(app):
def test_get_ips_list():
"""Test get IP list."""
ranges = ['127.0.0.1', '192.168.1.3-5', '12.13.14.15/32']
assert get_ips_list(ranges) == [
'12.13.14.15', '127.0.0.1', '192.168.1.3', '192.168.1.4', '192.168.1.5'
]
assert set(get_ips_list(ranges)) == set([
'12.13.14.15/32', '127.0.0.1/32', '192.168.1.3/32', '192.168.1.4/31'
])

0 comments on commit 1dece3b

Please sign in to comment.