Skip to content

Commit

Permalink
[SW-273] Remove the workaround introduced by SW-272 as it's already f…
Browse files Browse the repository at this point in the history
…ixed in Spark 2.1.0 and later (#529)

(cherry picked from commit 66685ed)
  • Loading branch information
jakubhava committed Jan 12, 2018
1 parent 3aa5adc commit c8337d0
Showing 1 changed file with 0 additions and 15 deletions.
15 changes: 0 additions & 15 deletions py/pysparkling/initializer.py
Expand Up @@ -48,21 +48,6 @@ def __add_sparkling_jar_to_spark(sc):
# when they need to use the dependency.
jsc.addJar(sw_jar_file)

# SW-272: For yarn and cluster deployments we need to modify location of the jar file
# to a driver cache location
#
# WARNING: This is low-level code accessing internal Spark API to enable SW2.0
# on Azure/CDH clusters.
# It needs to be properly tested and verified on different platforms!
if jsc.sc().master() == "yarn" and jsc.sc().deployMode() == "cluster":
file_server = jsc.env().rpcEnv().fileServer()
field_jars = file_server.getClass().getDeclaredField("jars")
if field_jars:
field_jars.setAccessible(True)
field_jars_value = field_jars.get(file_server)
field_jars_value.put('sparkling_water_assembly.jar', jvm.java.io.File(sw_jar_file))


@staticmethod
def __extracted_jar_path():
import sparkling_water
Expand Down

0 comments on commit c8337d0

Please sign in to comment.