Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

implemented groups in works #297

Merged
merged 2 commits into from
May 6, 2024
Merged

implemented groups in works #297

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c0f849e
[kahi_scienti_works] added group to works
omazapa May 5, 2024
f15a5a3
[kahi_scienti_works] impemented groups properly when work is updated
omazapa May 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 61 additions & 2 deletions Kahi_scienti_works/kahi_scienti_works/process_one.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ def process_one_update(scienti_reg, colav_reg, db, collection, empty_work, verbo
Register from the scienti database
colav_reg : dict
Register from the colav database (kahi database for impactu)
db: pymongo.collection.Collection
Database where ETL result is stored
collection : pymongo.collection.Collection
Collection in the database where the register is stored (Collection of works)
empty_work : dict
Expand All @@ -29,6 +31,28 @@ def process_one_update(scienti_reg, colav_reg, db, collection, empty_work, verbo
authors_ids = []
for upd in colav_reg["updated"]:
if upd["source"] == "scienti":
# scienti groups
if "group" in scienti_reg.keys():
for group in scienti_reg["group"]:
group_reg = db["affiliations"].find_one(
{"external_ids.id": group["COD_ID_GRUPO"]})
if group_reg is None:
group_reg = db["affiliations"].find_one(
{"external_ids.id": group["NRO_ID_GRUPO"]})
if group_reg:
found = False
for rgroup in colav_reg["groups"]:
if group_reg["_id"] == rgroup["id"]:
found = True
break
if not found:
colav_reg["groups"].append(
{"id": group_reg["_id"], "name": group_reg["names"][0]["name"]})
if not group_reg:
print(
f'WARNING: group with ids {scienti_reg["group"]["COD_ID_GRUPO"]} and {scienti_reg["group"]["NRO_ID_GRUPO"]} not found in affiliation')

# authors
authors_ids = [str(colav_author["id"])
for colav_author in colav_reg["authors"] if "id" in colav_author.keys()]
# adding new author and affiliations to the register
Expand Down Expand Up @@ -135,7 +159,7 @@ def process_one_update(scienti_reg, colav_reg, db, collection, empty_work, verbo
collection.update_one(
{"_id": colav_reg["_id"]},
{"$push": {"authors": entry['authors'][0]}, "$inc": {
"author_count": 1}}
"author_count": 1}, "$set": {"groups": colav_reg["groups"]}}
)

return None # Register already on db
Expand Down Expand Up @@ -203,7 +227,26 @@ def process_one_update(scienti_reg, colav_reg, db, collection, empty_work, verbo
"affiliations": author["affiliations"]
}
break

# scienti groups
if "group" in scienti_reg.keys():
for group in scienti_reg["group"]:
group_reg = db["affiliations"].find_one(
{"external_ids.id": group["COD_ID_GRUPO"]})
if group_reg is None:
group_reg = db["affiliations"].find_one(
{"external_ids.id": group["NRO_ID_GRUPO"]})
if group_reg:
found = False
for rgroup in colav_reg["groups"]:
if group_reg["_id"] == rgroup["id"]:
found = True
break
if not found:
colav_reg["groups"].append(
{"id": group_reg["_id"], "name": group_reg["names"][0]["name"]})
if not group_reg:
print(
f'WARNING: group with ids {scienti_reg["group"]["COD_ID_GRUPO"]} and {scienti_reg["group"]["NRO_ID_GRUPO"]} not found in affiliation')
collection.update_one(
{"_id": colav_reg["_id"]},
{"$set": {
Expand All @@ -215,6 +258,7 @@ def process_one_update(scienti_reg, colav_reg, db, collection, empty_work, verbo
"external_urls": colav_reg["external_urls"],
"authors": colav_reg["authors"],
"subjects": colav_reg["subjects"],
"groups": colav_reg["groups"]
}}
)

Expand Down Expand Up @@ -396,6 +440,21 @@ def process_one_insert(scienti_reg, db, collection, empty_work, es_handler, verb
"types": []
}
entry["author_count"] = len(entry["authors"])
# scienti group
if "group" in scienti_reg.keys():
for group in scienti_reg["group"]:
group_reg = db["affiliations"].find_one(
{"external_ids.id": group["COD_ID_GRUPO"]})
if group_reg is None:
group_reg = db["affiliations"].find_one(
{"external_ids.id": group["NRO_ID_GRUPO"]})
if group_reg:
entry["groups"].append(
{"id": group_reg["_id"], "name": group_reg["names"][0]["name"]})
if not group_reg:
print(
f'WARNING: group with ids {scienti_reg["group"]["COD_ID_GRUPO"]} and {scienti_reg["group"]["NRO_ID_GRUPO"]} not found in affiliation')

# insert in mongo
response = collection.insert_one(entry)
# insert in elasticsearch
Expand Down
Loading