Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Release/v8.0.1 #153

Merged
merged 7 commits into from
Apr 24, 2024
Merged

Release/v8.0.1 #153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
dee7eda
Merge pull request #149 from bcgsc/master
sshugsc Apr 11, 2024
ce9e5ef
fix pageToken bug
sshugsc Apr 16, 2024
27511ed
Merge pull request #150 from bcgsc/Bugfix/KBDEV-1200-ClinicalTrialsgo…
sshugsc Apr 18, 2024
76b7904
Update to CIViC API v2.4
mathieulemieux Apr 23, 2024
459369f
linting
mathieulemieux Apr 23, 2024
9052324
Merge pull request #152 from bcgsc/bugfix/KBDEV-1202-civic-loader-cro…
mathieulemieux Apr 24, 2024
bbb94d2
Bump version to v8.0.1
mathieulemieux Apr 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@bcgsc-pori/graphkb-loader",
"main": "src/index.js",
"version": "8.0.0",
"version": "8.0.1",
"repository": {
"type": "git",
"url": "https://github.com/bcgsc/pori_graphkb_loader.git"
Expand Down
14 changes: 10 additions & 4 deletions src/civic/evidenceItems.graphql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,15 +72,21 @@ query evidenceItems(
id
name
parsedName {
... on Gene { entrezId }
__typename
... on MolecularProfileTextSegment { text }
... on Variant { id }
}
rawName
variants {
gene {
entrezId
name
feature {
featureInstance {
__typename
... on Factor { id }
... on Gene {
entrezId
name
}
}
}
id
name
Expand Down
30 changes: 24 additions & 6 deletions src/civic/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,6 @@ const { EvidenceItem: evidenceSpec } = require('./specs.json');

class NotImplementedError extends ErrorMixin { }

const ajv = new Ajv();

const BASE_URL = 'https://civicdb.org/api/graphql';

/**
Expand All @@ -50,6 +48,8 @@ const VOCAB = {

const EVIDENCE_LEVEL_CACHE = {}; // avoid unecessary requests by caching the evidence levels

// Spec compiler
const ajv = new Ajv();
const validateEvidenceSpec = ajv.compile(evidenceSpec);


Expand Down Expand Up @@ -213,11 +213,28 @@ const processEvidenceRecord = async (opt) => {
conn, rawRecord, sources, variantsCache, oneToOne = false,
} = opt;

const [level, relevance, [feature]] = await Promise.all([
// Relevance & EvidenceLevel
const [level, relevance] = await Promise.all([
getEvidenceLevel(opt),
getRelevance(opt),
_entrezGene.fetchAndLoadByIds(conn, [rawRecord.variant.gene.entrezId]),
]);

// Variant's Feature
let feature;
const civicFeature = rawRecord.variant.feature.featureInstance;

if (civicFeature.__typename === 'Gene') {
[feature] = await _entrezGene.fetchAndLoadByIds(conn, [civicFeature.entrezId]);
} else if (civicFeature.__typename === 'Factor') {
// TODO: Deal with __typename === 'Factor'
// No actual case as April 22nd, 2024
throw new NotImplementedError(
'unable to process variant\'s feature of type Factor',
);
}


// Variant
let variants;

if (variantsCache.records[rawRecord.variant.id]) {
Expand All @@ -236,7 +253,6 @@ const processEvidenceRecord = async (opt) => {
}
}


// get the disease by doid
let disease;

Expand Down Expand Up @@ -382,6 +398,7 @@ const processEvidenceRecord = async (opt) => {
// update the existing record
return conn.updateRecord('Statement', rid(original), content);
}

// create a new record
return conn.addRecord({
content,
Expand Down Expand Up @@ -486,6 +503,7 @@ const downloadEvidenceRecords = async (url, trustedCurators) => {
}
records.push(record);
}
logger.info(`${records.length}/${evidenceItems.length} evidenceItem records successfully validated with the specs`);
return { counts, errorList, records };
};

Expand Down Expand Up @@ -604,7 +622,7 @@ const upload = async ({
record.conditions = Mp.process().conditions;
} catch (err) {
logger.error(`evidence (${record.id}) ${err}`);
counts.skip += 1;
counts.skip++;
continue;
}

Expand Down
4 changes: 2 additions & 2 deletions src/civic/profile.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,8 +226,8 @@ const MolecularProfile = (molecularProfile) => ({
`unable to process molecular profile with NOT operator (${this.profile.id || ''})`,
);
}
// Filters out unwanted gene's info from expression
const filteredParsedName = parsedName.filter(el => !el.entrezId);
// Filters out unwanted Feature info from expression
const filteredParsedName = parsedName.filter(el => el.__typename !== 'Feature');

// Parse expression into conditions
this.conditions = this._parse(filteredParsedName);
Expand Down
122 changes: 67 additions & 55 deletions src/civic/specs.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,32 +84,21 @@
},
"parsedName": {
"items": {
"anyOf": [
{
"properties": {
"entrezId": {
"type": "number"
}
},
"type": "object"
"properties": {
"__typename": {
"type": "string"
},
{
"properties": {
"id": {
"type": "number"
}
},
"type": "object"
"id": {
"type": "number"
},
{
"properties": {
"text": {
"type": "string"
}
},
"type": "object"
"text": {
"type": "string"
}
]
},
"required":[
"__typename"
],
"type": "object"
},
"type": "array"
},
Expand All @@ -122,39 +111,45 @@
"variants": {
"items": {
"properties": {
"gene": {
"feature": {
"properties": {
"entrezId": {
"type": [
"null",
"number"
]
},
"name": {
"type": [
"null",
"string"
]
"featureInstance": {
"properties": {
"__typename": {
"type": "string"
},
"entrezId": {
"type": "number"
},
"name": {
"type": "string"
}
},
"required":[
"__typename",
"entrezId",
"name"
],
"type": "object"
}
},
"type": [
"null",
"object"
]
"required": [
"featureInstance"
],
"type": "object"
},
"id": {
"type": [
"null",
"number"
]
"type": "number"
},
"name": {
"type": [
"null",
"string"
]
"type": "string"
}
},
"required": [
"feature",
"id",
"name"
],
"type": [
"null",
"object"
Expand All @@ -166,10 +161,14 @@
]
}
},
"type": [
"null",
"object"
]
"required": [
"id",
"name",
"parsedName",
"rawName",
"variants"
],
"type": "object"
},
"phenotypes": {
"items": {
Expand Down Expand Up @@ -317,9 +316,22 @@
]
}
},
"type": [
"null",
"object"
]
"required":[
"description",
"disease",
"evidenceDirection",
"evidenceLevel",
"evidenceRating",
"evidenceType",
"id",
"molecularProfile",
"phenotypes",
"significance",
"source",
"status",
"therapies",
"therapyInteractionType"
],
"type": "object"
}
}
39 changes: 27 additions & 12 deletions src/civic/variant.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
const kbParser = require('@bcgsc-pori/graphkb-parser');

const { error: { ParsingError } } = kbParser;
const {
rid,
} = require('../graphkb');
const { rid } = require('../graphkb');
const _entrezGene = require('../entrez/gene');
const _snp = require('../entrez/snp');
const { civic: SOURCE_DEFN } = require('../sources');

const {
civic: SOURCE_DEFN,
} = require('../sources');
const { error: { ErrorMixin, ParsingError } } = kbParser;
class NotImplementedError extends ErrorMixin { }


// based on discussion with cam here: https://www.bcgsc.ca/jira/browse/KBDEV-844
Expand Down Expand Up @@ -229,8 +225,8 @@ const normalizeVariantRecord = ({
};

/**
* Given some normalized variant record from CIViC load into graphkb, create links and
* return the record
* Given some normalized variant record from CIViC,
* load into graphkb, create links and return the record
*
* @param {ApiConnection} conn the connection to GraphKB
* @param {Object} normalizedVariant the normalized variant record
Expand Down Expand Up @@ -328,11 +324,30 @@ const uploadNormalizedVariant = async (conn, normalizedVariant, feature) => {

/**
* Given some variant record and a feature, process the variant and return a GraphKB equivalent
*
* @param {ApiConnection} conn the connection to GraphKB
* @param {Object} civicVariantRecord the raw variant record from CIViC
* @param {Object} feature the gene feature already grabbed from GraphKB
*/
const processVariantRecord = async (conn, civicVariantRecord, feature) => {
const featureInstance = civicVariantRecord.feature.featureInstance;
let entrezId,
entrezName;

if (featureInstance.__typename === 'Gene') {
entrezId = featureInstance.entrezId;
entrezName = featureInstance.name;
} else if (featureInstance.__typename === 'Factor') {
// TODO: Deal with __typename === 'Factor'
// No actual case as April 22nd, 2024
throw new NotImplementedError(
'unable to process variant\'s feature of type Factor',
);
}

const variants = normalizeVariantRecord({
entrezId: civicVariantRecord.gene.entrezId,
entrezName: civicVariantRecord.gene.name,
entrezId,
entrezName,
name: civicVariantRecord.name,
});

Expand Down
9 changes: 6 additions & 3 deletions src/clinicaltrialsgov/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,8 @@ const formatDate = (date) => `${date.getFullYear()}-${date.getMonth() + 1}-${dat
const upload = async ({ conn, maxRecords, days }) => {
const source = await conn.addSource(SOURCE_DEFN);

let options = {};
let options,
optionsWithToken;

if (days) {
const startDate = new Date(Date.now() - days * 24 * 60 * 60 * 1000);
Expand All @@ -320,7 +321,9 @@ const upload = async ({ conn, maxRecords, days }) => {

while (next) {
if (nextToken) {
options = { pageToken: nextToken, ...options };
optionsWithToken = { pageToken: nextToken, ...options };
} else {
optionsWithToken = options;
}
const trials = await requestWithRetry({
json: true,
Expand All @@ -331,7 +334,7 @@ const upload = async ({ conn, maxRecords, days }) => {
pageSize: 1000,
'query.cond': 'cancer',
sort: 'LastUpdatePostDate',
...options,
...optionsWithToken,
},
uri: BASE_URL,
});
Expand Down
Loading
Loading