Skip to content

Commit

Permalink
add note
Browse files Browse the repository at this point in the history
  • Loading branch information
@alimanfoo
alimanfoo committed Sep 12, 2016
1 parent f722f47 commit 510cbcc
Show file tree
Hide file tree
Showing 2 changed files with 172 additions and 8 deletions.
90 changes: 86 additions & 4 deletions notebooks/.ipynb_checkpoints/dask_2d_subset-checkpoint.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -755,12 +755,94 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 30,
"metadata": {
"collapsed": true
"collapsed": false
},
"outputs": [],
"source": []
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" 80055494 function calls (60052157 primitive calls) in 19.425 seconds\n",
"\n",
" Ordered by: internal time\n",
"\n",
" ncalls tottime percall cumtime percall filename:lineno(function)\n",
"10001670/3 5.032 0.000 6.671 2.224 slicing.py:623(check_index)\n",
" 1 3.459 3.459 4.272 4.272 slicing.py:398(partition_by_size)\n",
"10001671/1 3.287 0.000 6.378 6.378 slicing.py:540(posify_index)\n",
" 40006704 2.999 0.000 2.999 0.000 {built-in method builtins.isinstance}\n",
" 1 1.731 1.731 6.378 6.378 slicing.py:563(<listcomp>)\n",
" 1 0.849 0.849 0.849 0.849 slicing.py:44(<listcomp>)\n",
" 10011685 0.433 0.000 0.433 0.000 {built-in method builtins.len}\n",
" 10015670 0.381 0.000 0.381 0.000 {method 'append' of 'list' objects}\n",
" 1 0.355 0.355 0.355 0.355 slicing.py:420(issorted)\n",
" 1 0.196 0.196 0.196 0.196 {method 'tolist' of 'numpy.ndarray' objects}\n",
" 1 0.193 0.193 0.193 0.193 slicing.py:479(<listcomp>)\n",
" 1 0.157 0.157 0.157 0.157 {built-in method builtins.sorted}\n",
" 1 0.085 0.085 4.707 4.707 slicing.py:441(take_sorted)\n",
" 1 0.085 0.085 19.425 19.425 <string>:1(<module>)\n",
" 1 0.079 0.079 19.341 19.341 core.py:1024(__getitem__)\n",
" 1 0.034 0.034 18.157 18.157 slicing.py:142(slice_with_newaxes)\n",
" 2 0.033 0.017 0.033 0.017 {built-in method _hashlib.openssl_md5}\n",
" 1 0.026 0.026 1.071 1.071 slicing.py:15(sanitize_index)\n",
" 4001 0.007 0.000 0.009 0.000 slicing.py:567(insert_many)\n",
" 2001 0.002 0.000 0.011 0.000 slicing.py:156(<genexpr>)\n",
" 12003 0.001 0.000 0.001 0.000 {method 'pop' of 'list' objects}\n",
" 4 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n",
" 1 0.000 0.000 19.228 19.228 slicing.py:60(slice_array)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:464(<listcomp>)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:162(<genexpr>)\n",
" 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n",
" 1 0.000 0.000 18.111 18.111 slicing.py:170(slice_wrap_lists)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:465(<listcomp>)\n",
" 1 0.000 0.000 5.062 5.062 slicing.py:487(take)\n",
" 1 0.000 0.000 0.033 0.033 base.py:343(tokenize)\n",
" 1 0.000 0.000 19.425 19.425 {built-in method builtins.exec}\n",
" 2 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n",
" 3 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n",
" 2 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n",
" 1 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n",
" 2 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n",
" 4 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n",
" 2 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n",
" 1 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n",
" 1 0.000 0.000 0.000 0.000 core.py:794(__init__)\n",
" 4 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:467(<listcomp>)\n",
" 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n",
" 4 0.000 0.000 0.000 0.000 slicing.py:183(<genexpr>)\n",
" 2 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:192(<listcomp>)\n",
" 4 0.000 0.000 0.000 0.000 slicing.py:207(<genexpr>)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:475(<listcomp>)\n",
" 2 0.000 0.000 0.033 0.017 utils.py:502(__call__)\n",
" 2 0.000 0.000 0.000 0.000 slicing.py:125(<genexpr>)\n",
" 2 0.000 0.000 0.000 0.000 core.py:1043(<genexpr>)\n",
" 4 0.000 0.000 0.000 0.000 slicing.py:197(<genexpr>)\n",
" 1 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n",
" 2 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n",
" 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n",
" 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n",
" 1 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:613(<listcomp>)\n",
" 1 0.000 0.000 0.000 0.000 core.py:1452(<lambda>)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:149(<listcomp>)\n",
" 2 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n",
" 2 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:150(<listcomp>)\n",
" 1 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n",
"\n",
"\n"
]
}
],
"source": [
"# problem is in fact just the dim0 selection\n",
"cProfile.run('d[c]', sort='time')"
]
}
],
"metadata": {
Expand Down
90 changes: 86 additions & 4 deletions notebooks/dask_2d_subset.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -755,12 +755,94 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 30,
"metadata": {
"collapsed": true
"collapsed": false
},
"outputs": [],
"source": []
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" 80055494 function calls (60052157 primitive calls) in 19.425 seconds\n",
"\n",
" Ordered by: internal time\n",
"\n",
" ncalls tottime percall cumtime percall filename:lineno(function)\n",
"10001670/3 5.032 0.000 6.671 2.224 slicing.py:623(check_index)\n",
" 1 3.459 3.459 4.272 4.272 slicing.py:398(partition_by_size)\n",
"10001671/1 3.287 0.000 6.378 6.378 slicing.py:540(posify_index)\n",
" 40006704 2.999 0.000 2.999 0.000 {built-in method builtins.isinstance}\n",
" 1 1.731 1.731 6.378 6.378 slicing.py:563(<listcomp>)\n",
" 1 0.849 0.849 0.849 0.849 slicing.py:44(<listcomp>)\n",
" 10011685 0.433 0.000 0.433 0.000 {built-in method builtins.len}\n",
" 10015670 0.381 0.000 0.381 0.000 {method 'append' of 'list' objects}\n",
" 1 0.355 0.355 0.355 0.355 slicing.py:420(issorted)\n",
" 1 0.196 0.196 0.196 0.196 {method 'tolist' of 'numpy.ndarray' objects}\n",
" 1 0.193 0.193 0.193 0.193 slicing.py:479(<listcomp>)\n",
" 1 0.157 0.157 0.157 0.157 {built-in method builtins.sorted}\n",
" 1 0.085 0.085 4.707 4.707 slicing.py:441(take_sorted)\n",
" 1 0.085 0.085 19.425 19.425 <string>:1(<module>)\n",
" 1 0.079 0.079 19.341 19.341 core.py:1024(__getitem__)\n",
" 1 0.034 0.034 18.157 18.157 slicing.py:142(slice_with_newaxes)\n",
" 2 0.033 0.017 0.033 0.017 {built-in method _hashlib.openssl_md5}\n",
" 1 0.026 0.026 1.071 1.071 slicing.py:15(sanitize_index)\n",
" 4001 0.007 0.000 0.009 0.000 slicing.py:567(insert_many)\n",
" 2001 0.002 0.000 0.011 0.000 slicing.py:156(<genexpr>)\n",
" 12003 0.001 0.000 0.001 0.000 {method 'pop' of 'list' objects}\n",
" 4 0.000 0.000 0.000 0.000 {method 'update' of 'dict' objects}\n",
" 1 0.000 0.000 19.228 19.228 slicing.py:60(slice_array)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:464(<listcomp>)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:162(<genexpr>)\n",
" 1 0.000 0.000 0.033 0.033 base.py:314(normalize_array)\n",
" 1 0.000 0.000 18.111 18.111 slicing.py:170(slice_wrap_lists)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:465(<listcomp>)\n",
" 1 0.000 0.000 5.062 5.062 slicing.py:487(take)\n",
" 1 0.000 0.000 0.033 0.033 base.py:343(tokenize)\n",
" 1 0.000 0.000 19.425 19.425 {built-in method builtins.exec}\n",
" 2 0.000 0.000 0.000 0.000 functoolz.py:217(__call__)\n",
" 3 0.000 0.000 0.000 0.000 {built-in method builtins.sum}\n",
" 2 0.000 0.000 0.000 0.000 abc.py:178(__instancecheck__)\n",
" 1 0.000 0.000 0.000 0.000 core.py:1455(normalize_chunks)\n",
" 2 0.000 0.000 0.000 0.000 dicttoolz.py:19(merge)\n",
" 4 0.000 0.000 0.000 0.000 _weakrefset.py:70(__contains__)\n",
" 2 0.000 0.000 0.000 0.000 dicttoolz.py:11(_get_factory)\n",
" 1 0.000 0.000 0.000 0.000 exceptions.py:15(merge)\n",
" 1 0.000 0.000 0.000 0.000 core.py:794(__init__)\n",
" 4 0.000 0.000 0.000 0.000 {built-in method builtins.all}\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:467(<listcomp>)\n",
" 1 0.000 0.000 0.000 0.000 {method 'view' of 'numpy.ndarray' objects}\n",
" 4 0.000 0.000 0.000 0.000 slicing.py:183(<genexpr>)\n",
" 2 0.000 0.000 0.000 0.000 {method 'hexdigest' of '_hashlib.HASH' objects}\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:606(replace_ellipsis)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:192(<listcomp>)\n",
" 4 0.000 0.000 0.000 0.000 slicing.py:207(<genexpr>)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:475(<listcomp>)\n",
" 2 0.000 0.000 0.033 0.017 utils.py:502(__call__)\n",
" 2 0.000 0.000 0.000 0.000 slicing.py:125(<genexpr>)\n",
" 2 0.000 0.000 0.000 0.000 core.py:1043(<genexpr>)\n",
" 4 0.000 0.000 0.000 0.000 slicing.py:197(<genexpr>)\n",
" 1 0.000 0.000 0.000 0.000 core.py:826(_get_chunks)\n",
" 2 0.000 0.000 0.000 0.000 {built-in method builtins.hasattr}\n",
" 1 0.000 0.000 0.000 0.000 {method 'ravel' of 'numpy.ndarray' objects}\n",
" 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects}\n",
" 1 0.000 0.000 0.000 0.000 {method 'encode' of 'str' objects}\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:613(<listcomp>)\n",
" 1 0.000 0.000 0.000 0.000 core.py:1452(<lambda>)\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:149(<listcomp>)\n",
" 2 0.000 0.000 0.000 0.000 {method 'pop' of 'dict' objects}\n",
" 2 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects}\n",
" 1 0.000 0.000 0.000 0.000 slicing.py:150(<listcomp>)\n",
" 1 0.000 0.000 0.000 0.000 {method 'count' of 'tuple' objects}\n",
"\n",
"\n"
]
}
],
"source": [
"# problem is in fact just the dim0 selection\n",
"cProfile.run('d[c]', sort='time')"
]
}
],
"metadata": {
Expand Down

0 comments on commit 510cbcc

Please sign in to comment.