-
Notifications
You must be signed in to change notification settings - Fork 1
/
bio_data_loader.py
215 lines (182 loc) · 8.81 KB
/
bio_data_loader.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
import os
import torch
import random
import networkx as nx
import pandas as pd
import numpy as np
from torch.utils import data
from torch_geometric.data import Data
from torch_geometric.data import InMemoryDataset
from torch_geometric.data import Batch
from itertools import repeat, product, chain
from collections import Counter, deque
from networkx.algorithms.traversal.breadth_first_search import generic_bfs_edges
def nx_to_graph_data_obj(g, center_id, allowable_features_downstream=None,
allowable_features_pretrain=None,
node_id_to_go_labels=None):
"""
Converts nx graph of PPI to pytorch geometric Data object.
:param g: nx graph object of ego graph
:param center_id: node id of center node in the ego graph
:param allowable_features_downstream: list of possible go function node
features for the downstream task. The resulting go_target_downstream node
feature vector will be in this order.
:param allowable_features_pretrain: list of possible go function node
features for the pretraining task. The resulting go_target_pretrain node
feature vector will be in this order.
:param node_id_to_go_labels: dict that maps node id to a list of its
corresponding go labels
:return: pytorch geometric Data object with the following attributes:
edge_attr
edge_index
x
species_id
center_node_idx
go_target_downstream (only if node_id_to_go_labels is not None)
go_target_pretrain (only if node_id_to_go_labels is not None)
"""
n_nodes = g.number_of_nodes()
n_edges = g.number_of_edges()
# nodes
nx_node_ids = [n_i for n_i in g.nodes()] # contains list of nx node ids
# in a particular ordering. Will be used as a mapping to convert
# between nx node ids and data obj node indices
x = torch.tensor(np.ones(n_nodes).reshape(-1, 1), dtype=torch.float)
# we don't have any node labels, so set to dummy 1. dim n_nodes x 1
center_node_idx = nx_node_ids.index(center_id)
center_node_idx = torch.tensor([center_node_idx], dtype=torch.long)
# edges
edges_list = []
edge_features_list = []
for node_1, node_2, attr_dict in g.edges(data=True):
edge_feature = [attr_dict['w1'], attr_dict['w2'], attr_dict['w3'],
attr_dict['w4'], attr_dict['w5'], attr_dict['w6'],
attr_dict['w7'], 0, 0] # last 2 indicate self-loop
# and masking
edge_feature = np.array(edge_feature, dtype=int)
# convert nx node ids to data obj node index
i = nx_node_ids.index(node_1)
j = nx_node_ids.index(node_2)
edges_list.append((i, j))
edge_features_list.append(edge_feature)
edges_list.append((j, i))
edge_features_list.append(edge_feature)
# data.edge_index: Graph connectivity in COO format with shape [2, num_edges]
edge_index = torch.tensor(np.array(edges_list).T, dtype=torch.long)
# data.edge_attr: Edge feature matrix with shape [num_edges, num_edge_features]
edge_attr = torch.tensor(np.array(edge_features_list),
dtype=torch.float)
try:
species_id = int(nx_node_ids[0].split('.')[0]) # nx node id is of the form:
# species_id.protein_id
species_id = torch.tensor([species_id], dtype=torch.long)
except: # occurs when nx node id has no species id info. For the extract
# substructure context pair transform, where we convert a data obj to
# a nx graph obj (which does not have original node id info)
species_id = torch.tensor([0], dtype=torch.long) # dummy species
# id is 0
# construct data obj
data = Data(x=x, edge_index=edge_index, edge_attr=edge_attr)
data.species_id = species_id
data.center_node_idx = center_node_idx
if node_id_to_go_labels: # supervised case with go node labels
# Construct a dim n_pretrain_go_classes tensor and a
# n_downstream_go_classes tensor for the center node. 0 is no data
# or negative, 1 is positive.
downstream_go_node_feature = [0] * len(allowable_features_downstream)
pretrain_go_node_feature = [0] * len(allowable_features_pretrain)
if center_id in node_id_to_go_labels:
go_labels = node_id_to_go_labels[center_id]
# get indices of allowable_features_downstream that match with elements
# in go_labels
_, node_feature_indices, _ = np.intersect1d(
allowable_features_downstream, go_labels, return_indices=True)
for idx in node_feature_indices:
downstream_go_node_feature[idx] = 1
# get indices of allowable_features_pretrain that match with
# elements in go_labels
_, node_feature_indices, _ = np.intersect1d(
allowable_features_pretrain, go_labels, return_indices=True)
for idx in node_feature_indices:
pretrain_go_node_feature[idx] = 1
data.go_target_downstream = torch.tensor(np.array(downstream_go_node_feature),
dtype=torch.long)
data.go_target_pretrain = torch.tensor(np.array(pretrain_go_node_feature),
dtype=torch.long)
return data
def graph_data_obj_to_nx(data):
"""
Converts pytorch geometric Data obj to network x data object.
:param data: pytorch geometric Data object
:return: nx graph object
"""
G = nx.Graph()
# edges
edge_index = data.edge_index.cpu().numpy()
edge_attr = data.edge_attr.cpu().numpy()
n_edges = edge_index.shape[1]
for j in range(0, n_edges, 2):
begin_idx = int(edge_index[0, j])
end_idx = int(edge_index[1, j])
w1, w2, w3, w4, w5, w6, w7, _, _ = edge_attr[j].astype(bool)
if not G.has_edge(begin_idx, end_idx):
G.add_edge(begin_idx, end_idx, w1=w1, w2=w2, w3=w3, w4=w4, w5=w5,
w6=w6, w7=w7)
# # add center node id information in final nx graph object
# nx.set_node_attributes(G, {data.center_node_idx.item(): True}, 'is_centre')
return G
class BioDataset(InMemoryDataset):
def __init__(self,
root,
data_type,
empty=False,
transform=None,
pre_transform=None,
pre_filter=None,
map_device="cpu"):
"""
Adapted from qm9.py. Disabled the download functionality
:param root: the data directory that contains a raw and processed dir
:param data_type: either supervised or unsupervised
:param empty: if True, then will not load any data obj. For
initializing empty dataset
:param transform:
:param pre_transform:
:param pre_filter:
"""
self.root = root
self.data_type = data_type
super(BioDataset, self).__init__(root, transform, pre_transform, pre_filter)
if not empty:
self.data, self.slices = torch.load(self.processed_paths[0], map_location=map_device) # default is load to gpu without setted map_location
@property
def raw_file_names(self):
#raise NotImplementedError('Data is assumed to be processed')
if self.data_type == 'supervised': # 8 labelled species
file_name_list = ['3702', '6239', '511145', '7227', '9606', '10090', '4932', '7955']
else: # unsupervised: 8 labelled species, and 42 top unlabelled species by n_nodes.
file_name_list = ['3702', '6239', '511145', '7227', '9606', '10090',
'4932', '7955', '3694', '39947', '10116', '443255', '9913', '13616',
'3847', '4577', '8364', '9823', '9615', '9544', '9796', '3055', '7159',
'9031', '7739', '395019', '88036', '9685', '9258', '9598', '485913',
'44689', '9593', '7897', '31033', '749414', '59729', '536227', '4081',
'8090', '9601', '749927', '13735', '448385', '457427', '3711', '479433',
'479432', '28377', '9646']
return file_name_list
@property
def processed_file_names(self):
return 'geometric_data_processed.pt'
def download(self):
raise NotImplementedError('Must indicate valid location of raw data. '
'No download allowed')
def process(self):
raise NotImplementedError('Data is assumed to be processed')
if __name__ == "__main__":
# root_supervised = 'dataset/supervised'
root_supervised = os.path.join(os.path.expanduser('~'), 'datasets/bio_dataset', 'supervised')
d_supervised = BioDataset(root_supervised, data_type='supervised')
print(d_supervised)
# root_unsupervised = 'dataset/unsupervised'
root_unsupervised = os.path.join(os.path.expanduser('~'), 'datasets/bio_dataset', 'unsupervised')
d_unsupervised = BioDataset(root_unsupervised, data_type='unsupervised')
print(d_unsupervised)