Skip to content

Commit

Permalink
sync
Browse files Browse the repository at this point in the history
  • Loading branch information
@andrewbolster
andrewbolster committed Oct 31, 2018
1 parent a8f593b commit 5801131
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 74 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1 +1,3 @@
.ipynb_checkpoints
.DS_store
.idea/**
146 changes: 78 additions & 68 deletions 2018-10 Dask/2018-10-PyBelfast.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,11 +210,11 @@
},
{
"cell_type": "code",
"execution_count": 222,
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T16:57:20.575144Z",
"start_time": "2018-10-11T16:57:20.418940Z"
"end_time": "2018-10-17T13:34:18.444622Z",
"start_time": "2018-10-17T13:34:15.672823Z"
},
"slideshow": {
"slide_type": "-"
Expand Down Expand Up @@ -248,7 +248,7 @@
"<Client: scheduler='tcp://10.244.7.6:8786' processes=7 cores=14>"
]
},
"execution_count": 222,
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -1224,11 +1224,11 @@
},
{
"cell_type": "code",
"execution_count": 242,
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:20:47.811681Z",
"start_time": "2018-10-11T18:20:46.917016Z"
"end_time": "2018-10-17T13:35:04.788185Z",
"start_time": "2018-10-17T13:34:56.554483Z"
},
"slideshow": {
"slide_type": "fragment"
Expand Down Expand Up @@ -1337,24 +1337,26 @@
"Dask Name: read-parquet, 100 tasks"
]
},
"execution_count": 242,
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from dask import dataframe\n",
"\n",
"df = dataframe.read_parquet(\"s3://datasets.farsetlabs.org.uk/parquet_dask_example/\", storage_options=dict(anon=True))\n",
"df = client.persist(df) # Runs Above and updates state\n",
"df"
]
},
{
"cell_type": "code",
"execution_count": 243,
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:21:04.201544Z",
"start_time": "2018-10-11T18:21:03.632535Z"
"end_time": "2018-10-17T13:35:40.085008Z",
"start_time": "2018-10-17T13:35:34.896298Z"
},
"slideshow": {
"slide_type": "subslide"
Expand All @@ -1373,7 +1375,7 @@
"Name: count, dtype: int64"
]
},
"execution_count": 243,
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -1385,11 +1387,11 @@
},
{
"cell_type": "code",
"execution_count": 244,
"execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:21:40.352973Z",
"start_time": "2018-10-11T18:21:40.298097Z"
"end_time": "2018-10-17T13:35:57.451625Z",
"start_time": "2018-10-17T13:35:57.433982Z"
},
"slideshow": {
"slide_type": "subslide"
Expand All @@ -1407,7 +1409,7 @@
"Dask Name: series-groupby-sum-agg, 216 tasks"
]
},
"execution_count": 244,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -1450,11 +1452,11 @@
},
{
"cell_type": "code",
"execution_count": 245,
"execution_count": 6,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:22:11.134723Z",
"start_time": "2018-10-11T18:22:09.054362Z"
"end_time": "2018-10-17T13:36:21.252798Z",
"start_time": "2018-10-17T13:36:08.242729Z"
},
"slideshow": {
"slide_type": "subslide"
Expand Down Expand Up @@ -1691,7 +1693,7 @@
"[5 rows x 2747 columns]"
]
},
"execution_count": 245,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -1702,11 +1704,11 @@
},
{
"cell_type": "code",
"execution_count": 246,
"execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:22:35.180241Z",
"start_time": "2018-10-11T18:22:35.097107Z"
"end_time": "2018-10-17T13:36:29.719815Z",
"start_time": "2018-10-17T13:36:29.675154Z"
},
"slideshow": {
"slide_type": "subslide"
Expand Down Expand Up @@ -1761,11 +1763,11 @@
},
{
"cell_type": "code",
"execution_count": 247,
"execution_count": 8,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:23:44.903371Z",
"start_time": "2018-10-11T18:23:43.482237Z"
"end_time": "2018-10-17T13:36:49.607679Z",
"start_time": "2018-10-17T13:36:46.322007Z"
},
"slideshow": {
"slide_type": "subslide"
Expand Down Expand Up @@ -2237,7 +2239,7 @@
"[100 rows x 3 columns]"
]
},
"execution_count": 247,
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -2260,11 +2262,11 @@
},
{
"cell_type": "code",
"execution_count": 248,
"execution_count": 9,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:24:12.697634Z",
"start_time": "2018-10-11T18:24:12.511230Z"
"end_time": "2018-10-17T13:37:07.984948Z",
"start_time": "2018-10-17T13:37:03.952673Z"
},
"slideshow": {
"slide_type": "-"
Expand All @@ -2277,7 +2279,7 @@
"dask.array<normal, shape=(100000, 20), dtype=float64, chunksize=(1000, 20)>"
]
},
"execution_count": 248,
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -2293,11 +2295,11 @@
},
{
"cell_type": "code",
"execution_count": 249,
"execution_count": 10,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:24:18.287538Z",
"start_time": "2018-10-11T18:24:16.871983Z"
"end_time": "2018-10-17T13:37:08.461871Z",
"start_time": "2018-10-17T13:37:07.988457Z"
},
"slideshow": {
"slide_type": "subslide"
Expand All @@ -2312,11 +2314,11 @@
},
{
"cell_type": "code",
"execution_count": 250,
"execution_count": 11,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:24:22.545712Z",
"start_time": "2018-10-11T18:24:22.329201Z"
"end_time": "2018-10-17T13:37:19.609087Z",
"start_time": "2018-10-17T13:37:08.526951Z"
},
"slideshow": {
"slide_type": "subslide"
Expand All @@ -2329,7 +2331,7 @@
"array([0, 1])"
]
},
"execution_count": 250,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -2342,11 +2344,11 @@
},
{
"cell_type": "code",
"execution_count": 251,
"execution_count": 12,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:24:27.600296Z",
"start_time": "2018-10-11T18:24:27.593526Z"
"end_time": "2018-10-17T13:37:19.691616Z",
"start_time": "2018-10-17T13:37:19.613139Z"
},
"slideshow": {
"slide_type": "subslide"
Expand All @@ -2364,7 +2366,7 @@
" validation_fraction=0.1, verbose=0, warm_start=False)"
]
},
"execution_count": 251,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -2378,11 +2380,11 @@
},
{
"cell_type": "code",
"execution_count": 252,
"execution_count": 13,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:24:40.758474Z",
"start_time": "2018-10-11T18:24:40.751757Z"
"end_time": "2018-10-17T13:37:19.721780Z",
"start_time": "2018-10-17T13:37:19.694204Z"
},
"slideshow": {
"slide_type": "subslide"
Expand All @@ -2401,7 +2403,7 @@
" random_state=None, scoring='accuracy', shuffle_blocks=True)"
]
},
"execution_count": 252,
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -2415,11 +2417,11 @@
},
{
"cell_type": "code",
"execution_count": 253,
"execution_count": 14,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:24:48.986111Z",
"start_time": "2018-10-11T18:24:47.877669Z"
"end_time": "2018-10-17T13:37:24.736895Z",
"start_time": "2018-10-17T13:37:19.724846Z"
},
"slideshow": {
"slide_type": "subslide"
Expand All @@ -2438,7 +2440,7 @@
" random_state=None, scoring='accuracy', shuffle_blocks=True)"
]
},
"execution_count": 253,
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -2449,11 +2451,11 @@
},
{
"cell_type": "code",
"execution_count": 254,
"execution_count": 15,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:24:54.972103Z",
"start_time": "2018-10-11T18:24:54.462062Z"
"end_time": "2018-10-17T13:37:27.912746Z",
"start_time": "2018-10-17T13:37:24.739485Z"
},
"slideshow": {
"slide_type": "fragment"
Expand All @@ -2463,10 +2465,10 @@
{
"data": {
"text/plain": [
"0.6768"
"0.6731333333333334"
]
},
"execution_count": 254,
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -2477,11 +2479,11 @@
},
{
"cell_type": "code",
"execution_count": 255,
"execution_count": 16,
"metadata": {
"ExecuteTime": {
"end_time": "2018-10-11T18:25:21.339461Z",
"start_time": "2018-10-11T18:25:05.058600Z"
"end_time": "2018-10-17T13:38:35.981972Z",
"start_time": "2018-10-17T13:37:27.915904Z"
},
"slideshow": {
"slide_type": "subslide"
Expand All @@ -2492,16 +2494,24 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Score: 0.6682666666666667\n",
"Score: 0.714\n",
"Score: 0.7059333333333333\n",
"Score: 0.7072\n",
"Score: 0.7107333333333333\n",
"Score: 0.7173333333333334\n",
"Score: 0.7190666666666666\n",
"Score: 0.7206666666666667\n",
"Score: 0.7188666666666667\n",
"Score: 0.7194666666666667\n"
"Score: 0.6832\n",
"Score: 0.6918666666666666\n",
"Score: 0.706\n",
"Score: 0.6896\n",
"Score: 0.7096\n",
"Score: 0.7116\n",
"Score: 0.7172666666666667\n",
"Score: 0.7117333333333333\n",
"Score: 0.7178666666666667\n",
"Score: 0.7082\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"distributed.batched - INFO - Batched Comm Closed: in <closed TCP>: OSError: [Errno 49] Can't assign requested address\n",
"distributed.batched - INFO - Batched Comm Closed: in <closed TCP>: OSError: [Errno 49] Can't assign requested address\n"
]
}
],
Expand Down
Loading

0 comments on commit 5801131

Please sign in to comment.