Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added a pyspark example using tdigest to calculate percentiles for di…
…fferent keys.
- Loading branch information
Dave Matthews
committed
Jun 10, 2016
1 parent
364b150
commit 96e3767
Showing
1 changed file
with
25 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
from __future__ import print_function | ||
|
||
from random import random | ||
from operator import add | ||
from tdigest import TDigest | ||
|
||
data = sc.parallelize([(0, random()) for _ in range(1000)] + | ||
[(1, random() + 1) for _ in range(1000)] + | ||
[(2, random() + 2) for _ in range(1000)], 10) | ||
|
||
def initialise_digest(v): | ||
d = TDigest() | ||
d.update(v) | ||
return d | ||
|
||
def update_digest(d, v): | ||
d.update(v) | ||
return d | ||
|
||
percentiles = data\ | ||
.combineByKey(initialise_digest, update_digest, add)\ | ||
.map(lambda kv: (kv[0], kv[1].percentile(95)))\ | ||
.collect() | ||
|
||
print(percentiles) |