Skip to content
This repository
Browse code

Fix HDFS unpropagation

The unpropagate action with the HDFS adapter was not working because it
was using wrong variable names. Bug report and initial patch provided by
Brett Wu.

Closes #100.
  • Loading branch information...
commit 32d323d9db03f7cdb397504660157e5f901f01ab 1 parent 73173b5
Pierre Riteau priteau authored
16 control/src/python/workspacecontrol/defaults/imageprocurement/propagate_hdfs.py
@@ -102,10 +102,10 @@ def propagate(self, remote_source, local_absolute_target):
102 102 self.c.log.info("Transfer complete: %fs" % round(transfer_time))
103 103
104 104 def unpropagate(self, local_absolute_source, remote_target):
105   - self.c.log.info("HDFS unpropagation - local target: %s" % local_absolute_target)
106   - self.c.log.info("HDFS unpropagation - remote source: %s" % remote_source)
  105 + self.c.log.info("HDFS unpropagation - local source: %s" % local_absolute_source)
  106 + self.c.log.info("HDFS unpropagation - remote target: %s" % remote_target)
107 107
108   - cmd = self.__generate_hdfs_push_cmd(remote_source, local_absolute_target)
  108 + cmd = self.__generate_hdfs_push_cmd(local_absolute_source, remote_target)
109 109 self.c.log.info("Running HDFS command: %s" % cmd)
110 110 transfer_time = -time()
111 111 try:
@@ -127,11 +127,11 @@ def unpropagate(self, local_absolute_source, remote_target):
127 127 #--------------------------------------------------------------------------
128 128 # Private helper functions
129 129
130   - def __generate_hdfs_pull_cmd(self, remote_target, local_absolute_target):
  130 + def __generate_hdfs_pull_cmd(self, remote_source, local_absolute_target):
131 131 # Generate command in the form of:
132 132 # /path/to/hadoop/bin/hadoop fs -fs <files system uri> -copyToLocal <src> <localdst>
133 133 if not self.parsed_source_url:
134   - self.parsed_source_url = self.__parse_url(remote_target)
  134 + self.parsed_source_url = self.__parse_url(remote_source)
135 135
136 136 ops = [self.hadoop, "fs",
137 137 "-fs", self.parsed_source_url[0]+'://'+self.parsed_source_url[1],
@@ -139,15 +139,15 @@ def __generate_hdfs_pull_cmd(self, remote_target, local_absolute_target):
139 139 cmd = " ".join(ops)
140 140 return cmd
141 141
142   - def __generate_hdfs_push_cmd(self, remote_target, local_absolute_target):
  142 + def __generate_hdfs_push_cmd(self, local_absolute_source, remote_target):
143 143 # Generate command in the form of:
144 144 # /path/to/hadoop/bin/hadoop fs -fs <files system uri> -copyFromLocal <local> <dst>
145 145 if not self.parsed_dest_url:
146 146 self.parsed_dest_url = self.__parse_url(remote_target)
147 147
148 148 ops = [self.hadoop, "fs",
149   - "-fs", self.parsed_dest_url[0]+'://'+self.parsed_dest_url[1],
150   - "-copyFromLocal", local_absolute_target ,self.parsed_dest_url[2]]
  149 + "-fs", self.parsed_dest_url[0] + '://'+self.parsed_dest_url[1],
  150 + "-copyFromLocal", local_absolute_source, self.parsed_dest_url[2]]
151 151 cmd = " ".join(ops)
152 152 return cmd
153 153

0 comments on commit 32d323d

Please sign in to comment.
Something went wrong with that request. Please try again.