diff --git a/bin/migrate_acl_authz.py b/bin/migrate_acl_authz.py index 34810330..8a2116a7 100644 --- a/bin/migrate_acl_authz.py +++ b/bin/migrate_acl_authz.py @@ -64,13 +64,12 @@ def main(): q, IndexRecord.did, int(args.chunk_size), - start=getattr(args, "start_did") + start=getattr(args, "start_did"), ) for record in wq: if not record.acl: logger.info( - "record {} has no acl, setting authz to empty" - .format(record.did) + "record {} has no acl, setting authz to empty".format(record.did) ) record.authz = [] continue @@ -93,7 +92,7 @@ def main(): def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( - "--path", default="/var/www/indexd/", help="path to find local_settings.py", + "--path", default="/var/www/indexd/", help="path to find local_settings.py" ) parser.add_argument( "--sheepdog-db", dest="sheepdog", help="URI for the sheepdog database" @@ -102,11 +101,15 @@ def parse_args(): "--arborist-url", dest="arborist", help="URL for the arborist service" ) parser.add_argument( - "--chunk-size", dest="chunk_size", type=int, default=1000, + "--chunk-size", + dest="chunk_size", + type=int, + default=1000, help="number of records to process at once", ) parser.add_argument( - "--start-did", dest="start_did", + "--start-did", + dest="start_did", help="did to start at (records processed in lexographical order)", ) return parser.parse_args() @@ -135,17 +138,21 @@ def __init__(self, arborist_url, sheepdog_db=None): raise EnvironmentError( "couldn't connect to sheepdog db using the provided URI" ) - result = connection.execute("SELECT _props->>'name' as name from node_program;") + result = connection.execute( + "SELECT _props->>'name' as name from node_program;" + ) for row in result: self.programs.add(row["name"]) - result = connection.execute(""" + result = connection.execute( + """ SELECT project._props->>'name' AS name, program._props->>'name' AS program FROM node_project AS project JOIN edge_projectmemberofprogram AS edge ON edge.src_id = project.node_id JOIN node_program AS program ON edge.dst_id = program.node_id; - """) + """ + ) for row in result: self.projects[row["name"]] = row["program"] connection.close() @@ -180,9 +187,8 @@ def acl_to_authz(self, record): # if there's a * it should just be open. return early path = "/open" break - elif ( - not self.use_sheepdog_db - or (projects_found == 0 and self.is_program(acl_item)) + elif not self.use_sheepdog_db or ( + projects_found == 0 and self.is_program(acl_item) ): # if we don't have sheepdog we have to assume everything is a "program". # also, we only want to set the path to a program if we haven't found a @@ -203,8 +209,9 @@ def acl_to_authz(self, record): if not path: logger.error( - "couldn't get `authz` for record {}; setting as empty" - .format(record.did) + "couldn't get `authz` for record {}; setting as empty".format( + record.did + ) ) return None @@ -224,14 +231,17 @@ def acl_to_authz(self, record): response = requests.post(url, timeout=5, json=resource) except requests.exceptions.Timeout: logger.error( - "couldn't hit arborist to look up resource (timed out): {}".format(url) + "couldn't hit arborist to look up resource (timed out): {}".format( + url + ) ) raise EnvironmentError("couldn't reach arborist; request timed out") tag = None try: logger.debug( - "got {} from arborist: {}" - .format(response.status_code, response.json()) + "got {} from arborist: {}".format( + response.status_code, response.json() + ) ) if response.status_code == 409: # resource is already there, so we'll just take the tag @@ -259,18 +269,15 @@ def acl_to_authz(self, record): def column_windows(session, column, windowsize, start=None): - def int_for_range(start_id, end_id): if end_id: return and_(column >= start_id, column < end_id) else: return column >= start_id - q = ( - session - .query(column, func.row_number().over(order_by=column).label('rownum')) - .from_self(column) - ) + q = session.query( + column, func.row_number().over(order_by=column).label("rownum") + ).from_self(column) if start: q = q.filter(column >= start) if windowsize > 1: diff --git a/indexd/bulk/blueprint.py b/indexd/bulk/blueprint.py index 1d37599c..8f28a39c 100644 --- a/indexd/bulk/blueprint.py +++ b/indexd/bulk/blueprint.py @@ -43,7 +43,7 @@ def bulk_get_documents(): query = query.filter(IndexRecord.did.in_(ids)) docs = [q.to_document_dict() for q in query] - return json.dumps(docs), 200 + return flask.Response(json.dumps(docs), 200, mimetype="application/json") @blueprint.record diff --git a/requirements.txt b/requirements.txt index c4c995d5..06ca2987 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -flask==0.12.4 +flask==1.1.1 jsonschema==2.5.1 sqlalchemy==1.3.3 sqlalchemy-utils>=0.33.11 diff --git a/setup.py b/setup.py index c00ef578..2eddbe79 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ packages=find_packages(), package_data={"index": ["schemas/*"]}, install_requires=[ - "flask==0.12.4", + "flask==1.1.1", "jsonschema==2.5.1", "sqlalchemy==1.3.3", "sqlalchemy-utils>=0.33.11", diff --git a/tests/test_driver_alchemy_auth.py b/tests/test_driver_alchemy_auth.py index 7afbd3b4..d68dbfbf 100644 --- a/tests/test_driver_alchemy_auth.py +++ b/tests/test_driver_alchemy_auth.py @@ -23,7 +23,9 @@ def test_driver_init_does_not_create_records(): Tests for creation of records after driver init. Tests driver init does not have unexpected side-effects. """ - driver = SQLAlchemyAuthDriver("sqlite:///auth.sq3") # pylint: disable=unused-variable + driver = SQLAlchemyAuthDriver( + "sqlite:///auth.sq3" + ) # pylint: disable=unused-variable with sqlite3.connect("auth.sq3") as conn: diff --git a/tests/test_driver_alchemy_crud.py b/tests/test_driver_alchemy_crud.py index 824b8097..f859616b 100644 --- a/tests/test_driver_alchemy_crud.py +++ b/tests/test_driver_alchemy_crud.py @@ -64,7 +64,9 @@ def test_driver_init_does_not_create_record_hashes(): """ with sqlite3.connect("index.sq3") as conn: - driver = SQLAlchemyIndexDriver("sqlite:///index.sq3") # pylint: disable=unused-variable + driver = SQLAlchemyIndexDriver( + "sqlite:///index.sq3" + ) # pylint: disable=unused-variable count = conn.execute( """ diff --git a/tests/test_schema_migration.py b/tests/test_schema_migration.py index d6aeecc3..2d620708 100644 --- a/tests/test_schema_migration.py +++ b/tests/test_schema_migration.py @@ -77,12 +77,17 @@ def test_migrate_acls(client, user): "hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"}, } + # create the record res = client.post("/index/", json=data, headers=user) rec = res.json assert res.status_code == 200 + + # migrate with settings["config"]["INDEX"]["driver"].session as session: migrate_7(session) - res = client.get(rec["did"]) + + # check that the record has been migrated + res = client.get("/" + rec["did"]) rec = res.json assert res.status_code == 200 assert rec["acl"] == ["a", "b"]