Skip to content

Commit fa3a5c5

Browse files
Raghu GantiRaghu Ganti
Raghu Ganti
authored and
Raghu Ganti
committed
First commit
0 parents  commit fa3a5c5

18 files changed

+637
-0
lines changed

‎Ray graph experiments.ipynb‎

Lines changed: 343 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,343 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"id": "israeli-batch",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"import ray"
11+
]
12+
},
13+
{
14+
"cell_type": "code",
15+
"execution_count": 2,
16+
"id": "loose-projector",
17+
"metadata": {},
18+
"outputs": [],
19+
"source": [
20+
"ray.shutdown()"
21+
]
22+
},
23+
{
24+
"cell_type": "code",
25+
"execution_count": 3,
26+
"id": "renewable-western",
27+
"metadata": {},
28+
"outputs": [
29+
{
30+
"name": "stderr",
31+
"output_type": "stream",
32+
"text": [
33+
"2021-01-28 15:01:35,154\tINFO services.py:1166 -- View the Ray dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265\u001b[39m\u001b[22m\n"
34+
]
35+
},
36+
{
37+
"data": {
38+
"text/plain": [
39+
"{'node_ip_address': '192.168.1.37',\n",
40+
" 'raylet_ip_address': '192.168.1.37',\n",
41+
" 'redis_address': '192.168.1.37:6379',\n",
42+
" 'object_store_address': '/tmp/ray/session_2021-01-28_15-01-34_643480_33619/sockets/plasma_store',\n",
43+
" 'raylet_socket_name': '/tmp/ray/session_2021-01-28_15-01-34_643480_33619/sockets/raylet',\n",
44+
" 'webui_url': '127.0.0.1:8265',\n",
45+
" 'session_dir': '/tmp/ray/session_2021-01-28_15-01-34_643480_33619',\n",
46+
" 'metrics_export_port': 55854}"
47+
]
48+
},
49+
"execution_count": 3,
50+
"metadata": {},
51+
"output_type": "execute_result"
52+
}
53+
],
54+
"source": [
55+
"ray.init()"
56+
]
57+
},
58+
{
59+
"cell_type": "code",
60+
"execution_count": 4,
61+
"id": "scheduled-miami",
62+
"metadata": {},
63+
"outputs": [],
64+
"source": [
65+
"%config IPCompleter.use_jedi = False"
66+
]
67+
},
68+
{
69+
"cell_type": "code",
70+
"execution_count": 41,
71+
"id": "leading-sheriff",
72+
"metadata": {},
73+
"outputs": [],
74+
"source": [
75+
"import numpy as np\n",
76+
"from sklearn.preprocessing import FunctionTransformer\n",
77+
"from sklearn.preprocessing import Binarizer\n",
78+
"\n",
79+
"transformer = FunctionTransformer(np.log1p)\n",
80+
"binarizer = Binarizer(threshold=0.5)"
81+
]
82+
},
83+
{
84+
"cell_type": "code",
85+
"execution_count": 42,
86+
"id": "olive-usage",
87+
"metadata": {},
88+
"outputs": [],
89+
"source": [
90+
"import com.ibm.research.ray.graph.Datamodel as dm\n",
91+
"\n",
92+
"pipeline = dm.Pipeline()\n",
93+
"\n",
94+
"node_a = dm.Node('a', transformer)\n",
95+
"node_b = dm.Node('b', transformer)\n",
96+
"node_c = dm.Node('c', transformer)\n",
97+
"node_d = dm.Node('d', binarizer)\n",
98+
"\n",
99+
"pipeline.add_edge(node_a, node_b)\n",
100+
"pipeline.add_edge(node_b, node_c)\n",
101+
"pipeline.add_edge(node_b, node_d)"
102+
]
103+
},
104+
{
105+
"cell_type": "code",
106+
"execution_count": 34,
107+
"id": "behind-dairy",
108+
"metadata": {},
109+
"outputs": [],
110+
"source": [
111+
"X = np.array([0, 1, 2, 3])"
112+
]
113+
},
114+
{
115+
"cell_type": "code",
116+
"execution_count": 23,
117+
"id": "amended-gravity",
118+
"metadata": {},
119+
"outputs": [
120+
{
121+
"data": {
122+
"text/plain": [
123+
"array([0. , 0.42303586, 0.55461836, 0.62580029])"
124+
]
125+
},
126+
"execution_count": 23,
127+
"metadata": {},
128+
"output_type": "execute_result"
129+
}
130+
],
131+
"source": [
132+
"transformer.transform(transformer.transform(transformer.transform(X)))"
133+
]
134+
},
135+
{
136+
"cell_type": "code",
137+
"execution_count": 24,
138+
"id": "romance-sender",
139+
"metadata": {},
140+
"outputs": [],
141+
"source": [
142+
"xref = ray.put(X)"
143+
]
144+
},
145+
{
146+
"cell_type": "code",
147+
"execution_count": 35,
148+
"id": "handy-offset",
149+
"metadata": {},
150+
"outputs": [],
151+
"source": [
152+
"import com.ibm.research.ray.graph.Runtime as rt"
153+
]
154+
},
155+
{
156+
"cell_type": "code",
157+
"execution_count": 36,
158+
"id": "copyrighted-procurement",
159+
"metadata": {},
160+
"outputs": [],
161+
"source": [
162+
"in_args={node_a: [xref]}"
163+
]
164+
},
165+
{
166+
"cell_type": "code",
167+
"execution_count": 43,
168+
"id": "specialized-health",
169+
"metadata": {
170+
"scrolled": true
171+
},
172+
"outputs": [],
173+
"source": [
174+
"out_args = rt.execute_pipeline(pipeline, in_args)"
175+
]
176+
},
177+
{
178+
"cell_type": "code",
179+
"execution_count": 44,
180+
"id": "binding-praise",
181+
"metadata": {},
182+
"outputs": [
183+
{
184+
"name": "stdout",
185+
"output_type": "stream",
186+
"text": [
187+
"c\n",
188+
"d\n"
189+
]
190+
}
191+
],
192+
"source": [
193+
"for key in out_args.keys():\n",
194+
" print(str(key))"
195+
]
196+
},
197+
{
198+
"cell_type": "code",
199+
"execution_count": 45,
200+
"id": "living-destiny",
201+
"metadata": {},
202+
"outputs": [
203+
{
204+
"data": {
205+
"text/plain": [
206+
"[array([0. , 0.42303586, 0.55461836, 0.62580029])]"
207+
]
208+
},
209+
"execution_count": 45,
210+
"metadata": {},
211+
"output_type": "execute_result"
212+
}
213+
],
214+
"source": [
215+
"ray.get(out_args[node_c])"
216+
]
217+
},
218+
{
219+
"cell_type": "code",
220+
"execution_count": 46,
221+
"id": "vanilla-wrong",
222+
"metadata": {},
223+
"outputs": [
224+
{
225+
"ename": "RayTaskError(ValueError)",
226+
"evalue": "\u001b[36mray::execute_node()\u001b[39m (pid=33653, ip=192.168.1.37)\n File \"python/ray/_raylet.pyx\", line 484, in ray._raylet.execute_task\n File \"/Users/rganti/PycharmProjects/ray-graphs/com/ibm/research/ray/graph/Runtime.py\", line 9, in execute_node\n return node.get_transformer().transform(args)\n File \"/Users/rganti/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/preprocessing/_data.py\", line 2207, in transform\n reset=False)\n File \"/Users/rganti/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/base.py\", line 421, in _validate_data\n X = check_array(X, **check_params)\n File \"/Users/rganti/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/utils/validation.py\", line 63, in inner_f\n return f(*args, **kwargs)\n File \"/Users/rganti/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/utils/validation.py\", line 641, in check_array\n \"if it contains a single sample.\".format(array))\nValueError: Expected 2D array, got 1D array instead:\narray=[0. 0.52658903 0.74127631 0.86974169].\nReshape your data either using array.reshape(-1, 1) if your data has a single feature or array.reshape(1, -1) if it contains a single sample.",
227+
"output_type": "error",
228+
"traceback": [
229+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
230+
"\u001b[0;31mRayTaskError(ValueError)\u001b[0m Traceback (most recent call last)",
231+
"\u001b[0;32m<ipython-input-46-4fc04be03d7a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mray\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_args\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnode_d\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
232+
"\u001b[0;32m~/Library/Python/3.7/lib/python/site-packages/ray/worker.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(object_refs, timeout)\u001b[0m\n\u001b[1;32m 1426\u001b[0m \u001b[0mworker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore_worker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump_object_store_memory_usage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1427\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRayTaskError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1428\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_instanceof_cause\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1429\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1430\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
233+
"\u001b[0;31mRayTaskError(ValueError)\u001b[0m: \u001b[36mray::execute_node()\u001b[39m (pid=33653, ip=192.168.1.37)\n File \"python/ray/_raylet.pyx\", line 484, in ray._raylet.execute_task\n File \"/Users/rganti/PycharmProjects/ray-graphs/com/ibm/research/ray/graph/Runtime.py\", line 9, in execute_node\n return node.get_transformer().transform(args)\n File \"/Users/rganti/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/preprocessing/_data.py\", line 2207, in transform\n reset=False)\n File \"/Users/rganti/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/base.py\", line 421, in _validate_data\n X = check_array(X, **check_params)\n File \"/Users/rganti/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/utils/validation.py\", line 63, in inner_f\n return f(*args, **kwargs)\n File \"/Users/rganti/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/utils/validation.py\", line 641, in check_array\n \"if it contains a single sample.\".format(array))\nValueError: Expected 2D array, got 1D array instead:\narray=[0. 0.52658903 0.74127631 0.86974169].\nReshape your data either using array.reshape(-1, 1) if your data has a single feature or array.reshape(1, -1) if it contains a single sample."
234+
]
235+
}
236+
],
237+
"source": [
238+
"ray.get(out_args[node_d])"
239+
]
240+
},
241+
{
242+
"cell_type": "code",
243+
"execution_count": 48,
244+
"id": "decreased-showcase",
245+
"metadata": {},
246+
"outputs": [],
247+
"source": [
248+
"from sklearn.preprocessing import MinMaxScaler\n",
249+
"\n",
250+
"scaler = MinMaxScaler()"
251+
]
252+
},
253+
{
254+
"cell_type": "code",
255+
"execution_count": 49,
256+
"id": "objective-province",
257+
"metadata": {},
258+
"outputs": [
259+
{
260+
"ename": "NotFittedError",
261+
"evalue": "This MinMaxScaler instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator.",
262+
"output_type": "error",
263+
"traceback": [
264+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
265+
"\u001b[0;31mNotFittedError\u001b[0m Traceback (most recent call last)",
266+
"\u001b[0;32m<ipython-input-49-b8a40112f386>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mscaler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
267+
"\u001b[0;32m~/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/preprocessing/_data.py\u001b[0m in \u001b[0;36mtransform\u001b[0;34m(self, X)\u001b[0m\n\u001b[1;32m 430\u001b[0m \u001b[0mTransformed\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 431\u001b[0m \"\"\"\n\u001b[0;32m--> 432\u001b[0;31m \u001b[0mcheck_is_fitted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 433\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 434\u001b[0m X = self._validate_data(X, copy=self.copy, dtype=FLOAT_DTYPES,\n",
268+
"\u001b[0;32m~/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[0mextra_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 62\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mextra_args\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 65\u001b[0m \u001b[0;31m# extra_args > 0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
269+
"\u001b[0;32m~/PycharmProjects/ray-graphs/venv37/lib/python3.7/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36mcheck_is_fitted\u001b[0;34m(estimator, attributes, msg, all_or_any)\u001b[0m\n\u001b[1;32m 1039\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1040\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mattrs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1041\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mNotFittedError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'name'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1042\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
270+
"\u001b[0;31mNotFittedError\u001b[0m: This MinMaxScaler instance is not fitted yet. Call 'fit' with appropriate arguments before using this estimator."
271+
]
272+
}
273+
],
274+
"source": [
275+
"scaler.transform(X)"
276+
]
277+
},
278+
{
279+
"cell_type": "code",
280+
"execution_count": null,
281+
"id": "beautiful-extension",
282+
"metadata": {},
283+
"outputs": [],
284+
"source": []
285+
},
286+
{
287+
"cell_type": "code",
288+
"execution_count": null,
289+
"id": "abandoned-andorra",
290+
"metadata": {},
291+
"outputs": [],
292+
"source": [
293+
"if not l:\n",
294+
" print('True')"
295+
]
296+
},
297+
{
298+
"cell_type": "code",
299+
"execution_count": null,
300+
"id": "subject-pakistan",
301+
"metadata": {},
302+
"outputs": [],
303+
"source": []
304+
},
305+
{
306+
"cell_type": "code",
307+
"execution_count": null,
308+
"id": "chemical-makeup",
309+
"metadata": {},
310+
"outputs": [],
311+
"source": []
312+
},
313+
{
314+
"cell_type": "code",
315+
"execution_count": null,
316+
"id": "north-laundry",
317+
"metadata": {},
318+
"outputs": [],
319+
"source": []
320+
}
321+
],
322+
"metadata": {
323+
"kernelspec": {
324+
"display_name": "Python 3",
325+
"language": "python",
326+
"name": "python3"
327+
},
328+
"language_info": {
329+
"codemirror_mode": {
330+
"name": "ipython",
331+
"version": 3
332+
},
333+
"file_extension": ".py",
334+
"mimetype": "text/x-python",
335+
"name": "python",
336+
"nbconvert_exporter": "python",
337+
"pygments_lexer": "ipython3",
338+
"version": "3.7.9"
339+
}
340+
},
341+
"nbformat": 4,
342+
"nbformat_minor": 5
343+
}

‎com/__init__.py‎

Whitespace-only changes.

‎com/__pycache__/__init__.cpython-37.pyc‎

143 Bytes
Binary file not shown.

‎com/ibm/__init__.py‎

Whitespace-only changes.

‎com/ibm/__pycache__/__init__.cpython-37.pyc‎

147 Bytes
Binary file not shown.

‎com/ibm/research/__init__.py‎

Whitespace-only changes.

‎com/ibm/research/__pycache__/__init__.cpython-37.pyc‎

156 Bytes
Binary file not shown.

‎com/ibm/research/ray/__init__.py‎

Whitespace-only changes.

‎com/ibm/research/ray/__pycache__/__init__.cpython-37.pyc‎

160 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)