-
Notifications
You must be signed in to change notification settings - Fork 12
/
terraform.py
190 lines (157 loc) · 6.5 KB
/
terraform.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
import datetime
import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
from databricks_sync import log
class ImportStage:
def __init__(self, base_path: Path):
if not base_path.exists():
base_path.mkdir(parents=True, exist_ok=True)
self.__base_path = base_path
@property
def stage_dir(self):
return self.__base_path
def stage_files(self, from_dir: Path):
# Does not respect nesting
for dirpath, dnames, fnames in os.walk(from_dir):
for f in fnames:
src_path = os.path.join(dirpath, f)
shutil.copy(src_path, self.__base_path)
def stage_file(self, from_path: Path):
# Does not respect nesting
if from_path.exists():
shutil.copy(from_path, self.__base_path)
class TerraformCommandError(subprocess.CalledProcessError):
def __init__(self, ret_code, cmd, out, err):
super(TerraformCommandError, self).__init__(ret_code, cmd)
self.out = out
self.err = err
class Terraform:
BASE_COMMAND = ["terraform"]
def __init__(self, working_dir: str = None, is_env_vars_included=False):
self.is_env_vars_included = is_env_vars_included
self.working_dir = working_dir
def _cmd(self, cmds, *args, **kwargs):
capture_output = kwargs.pop('capture_output', True)
print_output = kwargs.pop("print_output", True)
# TODO maybe figure out where to set this and how to pass it here
raise_on_error = True
if capture_output is True:
stderr = subprocess.PIPE
stdout = subprocess.PIPE
else:
stderr = sys.stderr
stdout = sys.stdout
# cmds = self.generate_cmd_string(cmd, *args, **kwargs)
log.info('command: {c}'.format(c=' '.join(cmds)))
working_folder = self.working_dir if self.working_dir is not None else None
environ_vars = {}
if self.is_env_vars_included:
environ_vars = os.environ.copy()
p = subprocess.Popen(cmds, stdout=stdout, stderr=subprocess.STDOUT,
cwd=working_folder, env=environ_vars, close_fds=True)
output = []
error = []
for line in p.stdout:
content = re.sub(r'\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))', '', line.decode("utf-8").rstrip("\n"))
output.append(content)
if print_output is True:
log.info(line.decode("utf-8").rstrip("\n"))
# for line in p.stderr:
# content = re.sub(r'\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))', '', line.decode("utf-8").rstrip("\n"))
# error.append(content)
# log.error(line.decode("utf-8").rstrip("\n"))
p.communicate()
# Close buffers
try:
p.stdout.flush()
except Exception as e:
log.debug(str(e))
finally:
p.stdout.close()
ret_code = p.returncode
if capture_output is True:
out = "\n".join(output)
err = "\n".join(error)
else:
out = None
err = None
if ret_code != 0 and raise_on_error:
raise TerraformCommandError(
ret_code, ' '.join(cmds), out=out, err=out)
return ret_code, out, err
def version(self):
version_cmd = self.BASE_COMMAND + ["--version"]
return self._cmd(version_cmd)
def init(self):
version_cmd = self.BASE_COMMAND + ["init"]
return self._cmd(version_cmd)
def validate(self):
validate_cmd = self.BASE_COMMAND + ["validate"]
return self._cmd(validate_cmd)
@staticmethod
def is_import_lock():
return os.getenv("DATABRICKS_SYNC_IMPORT_LOCK", "false").lower()
@staticmethod
def get_import_plan_parallelism():
val = os.getenv("DATABRICKS_SYNC_IMPORT_PLAN_PARALLELISM", -1)
if isinstance(val, int):
return val
else:
return int(val)
@staticmethod
def get_import_apply_parallelism():
val = os.getenv("DATABRICKS_SYNC_IMPORT_APPLY_PARALLELISM", -1)
if isinstance(val, int):
return val
else:
return int(val)
def plan(self, output_file: Path = None, targets=None, state_file_abs_path: Path = None, refresh=None):
plan_cmd = self.BASE_COMMAND + ["plan"]
plan_cmd += [f"-lock={self.is_import_lock()}"]
if self.get_import_plan_parallelism() > 0:
plan_cmd += [f"-parallelism={str(self.get_import_plan_parallelism())}"]
if output_file is not None:
plan_cmd += ["-out", str(output_file.absolute())]
if state_file_abs_path is not None:
plan_cmd += ["-state", str(state_file_abs_path.absolute())]
if refresh is not None and refresh is False:
plan_cmd += ["-refresh=false"]
if targets is not None:
plan_cmd += targets
plan_cmd += ["-input=false"]
return self._cmd(plan_cmd)
@staticmethod
def __get_backup_path(state_file_abs_path: Path = None):
now_str = datetime.datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S_%f")
if state_file_abs_path is None:
return None
parent = state_file_abs_path.parent
file = state_file_abs_path.name
return parent / f"{str(file)}.{now_str}.bckup"
def apply(self, plan_file: Path = None, state_file_abs_path: Path = None, refresh=None):
apply_cmd = self.BASE_COMMAND + ["apply"]
backup_path = self.__get_backup_path(state_file_abs_path)
apply_cmd += [f"-lock={self.is_import_lock()}"]
if self.get_import_apply_parallelism() > 0:
apply_cmd += [f"-parallelism={str(self.get_import_apply_parallelism())}"]
if state_file_abs_path is not None:
apply_cmd += ["-state", str(state_file_abs_path.absolute())]
if backup_path is not None:
apply_cmd += ["-backup", str(backup_path.absolute())]
if refresh is not None and refresh is False:
apply_cmd += ["-refresh=false"]
if plan_file is not None:
apply_cmd += [str(plan_file.absolute())]
return self._cmd(apply_cmd)
def state_pull(self, state_file_abs_path: Path = None):
apply_cmd = self.BASE_COMMAND + ["state"]
if state_file_abs_path is not None:
apply_cmd += ["-state", str(state_file_abs_path.absolute())]
apply_cmd += ["pull"]
return self._cmd(apply_cmd, print_output=False)
def raw_cmd(self, command):
return self._cmd(command.split(" "))