Skip to content

Commit

Permalink
Python style fix
Browse files Browse the repository at this point in the history
  • Loading branch information
holdenk committed Apr 24, 2020
1 parent afb1b1a commit 4071ae2
Showing 1 changed file with 4 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,17 @@
.getOrCreate()
sc = spark._sc
acc = sc.accumulator(0)

def addToAcc(x):
acc.add(1)
return x

initialRdd = sc.parallelize(range(100), 5)
accRdd = initialRdd.map(addToAcc)
# Trigger a shuffle so there are shuffle blocks to migrate
rdd = accRdd.map(lambda x: (x, x)).groupByKey()
rdd.collect()
print("1st accumulator value is: "+ str(acc.value))
print("1st accumulator value is: " + str(acc.value))
print("Waiting to give nodes time to finish migration, decom exec 1.")
print("...")
time.sleep(5)
Expand All @@ -51,7 +53,7 @@ def addToAcc(x):
print("Executor node should be deleted now")
rdd.count()
rdd.collect()
print("Final accumulator value is: "+ str(acc.value))
print("Final accumulator value is: " + str(acc.value))
print("Finished waiting, stopping Spark.")
spark.stop()
print("Done, exiting Python")
Expand Down

0 comments on commit 4071ae2

Please sign in to comment.