Skip to content

Commit

Permalink
Improve/merge record_invalid_template
Browse files Browse the repository at this point in the history
  • Loading branch information
@KnVerey
KnVerey committed Apr 6, 2018
1 parent aca0780 commit 2348c89
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 50 deletions.
85 changes: 47 additions & 38 deletions lib/kubernetes-deploy/deploy_task.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,15 +187,6 @@ def run!(verify_result: true, allow_protected_ns: false, prune: true)

private

# Inspect the file referenced in the kubectl stderr
# to make it easier for developer to understand what's going on
def find_bad_files_from_kubectl_output(stderr)
# stderr often contains one or more lines like the following, from which we can extract the file path(s):
# Error from server (TypeOfError): error when creating "/path/to/service-gqq5oh.yml": Service "web" is invalid:
matches = stderr.scan(%r{"(/\S+\.ya?ml\S*)"})
matches&.flatten
end

def deploy_has_priority_resources?(resources)
resources.any? { |r| PREDEPLOY_SEQUENCE.include?(r.type) }
end
Expand Down Expand Up @@ -225,7 +216,8 @@ def validate_definitions(resources)
return unless failed_resources.present?

failed_resources.each do |r|
record_invalid_template(r.validation_error_msg, file_paths: [r.file_path])
content = File.read(r.file_path) if File.file?(r.file_path)
record_invalid_template(err: r.validation_error_msg, filename: File.basename(r.file_path), content: content)
end
raise FatalDeploymentError, "Template validation failed"
end
Expand All @@ -244,10 +236,6 @@ def discover_resources
end
end
resources
rescue InvalidTemplateError => e
debug_msg = paragraph_for_invalid_templates(e.message, filenames: [e.filename], content: e.content)
@logger.summary.add_paragraph(debug_msg)
raise FatalDeploymentError, "Failed to render and parse template"
end

def split_templates(filename)
Expand All @@ -261,29 +249,19 @@ def split_templates(filename)
end
yield doc
end
rescue InvalidTemplateError => e
record_invalid_template(err: e.message, filename: e.filename, content: e.content)
raise FatalDeploymentError, "Failed to render and parse template"
rescue Psych::SyntaxError => e
raise InvalidTemplateError.new(e, filename: filename, content: rendered_content)
end

def record_invalid_template(err, file_paths:, original_filenames: nil)
template_names = Array(original_filenames)
file_content = Array(file_paths).each_with_object([]) do |file_path, contents|
next unless File.file?(file_path)
contents << File.read(file_path)
template_names << File.basename(file_path) unless original_filenames
end.join("\n")
debug_msg = paragraph_for_invalid_templates(err, filenames: template_names, content: file_content)
@logger.summary.add_paragraph(debug_msg)
record_invalid_template(err: e.message, filename: filename, content: rendered_content)
raise FatalDeploymentError, "Failed to render and parse template"
end

def paragraph_for_invalid_templates(err, filenames:, content:)
template_list = filenames.compact.join(", ").presence || "See error message"
debug_msg = ColorizedString.new("Invalid #{'template'.pluralize(filenames.length)}: #{template_list}\n").red
def record_invalid_template(err:, filename:, content:)
debug_msg = ColorizedString.new("Invalid template: #{filename}\n").red
debug_msg += "> Error message:\n#{indent_four(err)}"
if content.present?
debug_msg += "\n> Template content:\n#{indent_four(content)}"
end
debug_msg
debug_msg += "\n> Template content:\n#{indent_four(content)}"
@logger.summary.add_paragraph(debug_msg)
end

def indent_four(str)
Expand Down Expand Up @@ -411,11 +389,7 @@ def apply_all(resources, prune)
if st.success?
log_pruning(out) if prune
else
file_paths = find_bad_files_from_kubectl_output(err)
warn_msg = "WARNING: Any resources not mentioned in the error below were likely created/updated. " \
"You may wish to roll back this deploy."
@logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)
record_invalid_template(err, file_paths: file_paths)
record_apply_failure(err)
raise FatalDeploymentError, "Command failed: #{Shellwords.join(command)}"
end
end
Expand All @@ -429,6 +403,41 @@ def log_pruning(kubectl_output)
@logger.summary.add_action("pruned #{pruned.length} #{'resource'.pluralize(pruned.length)}")
end

def record_apply_failure(err)
warn_msg = "WARNING: Any resources not mentioned in the error(s) below were likely created/updated. " \
"You may wish to roll back this deploy."
@logger.summary.add_paragraph(ColorizedString.new(warn_msg).yellow)

unidentified_errors = []
err.each_line do |line|
bad_files = find_bad_files_from_kubectl_output(line)
if bad_files.present?
bad_files.each { |f| record_invalid_template(err: f[:err], filename: f[:filename], content: f[:content]) }
else
unidentified_errors << line
end
end

if unidentified_errors.present?
msg = "#{ColorizedString.new('Unidentified error(s):').red}\n#{indent_four(unidentified_errors.join)}"
@logger.summary.add_paragraph(msg)
end
end

# Inspect the file referenced in the kubectl stderr
# to make it easier for developer to understand what's going on
def find_bad_files_from_kubectl_output(line)
# stderr often contains one or more lines like the following, from which we can extract the file path(s):
# Error from server (TypeOfError): error when creating "/path/to/service-gqq5oh.yml": Service "web" is invalid:

line.scan(%r{"(/\S+\.ya?ml\S*)"}).each_with_object([]) do |matches, bad_files|
matches.each do |path|
content = File.read(path) if File.file?(path)
bad_files << { filename: File.basename(path), err: line, content: content }
end
end
end

def confirm_context_exists
out, err, st = kubectl.run("config", "get-contexts", "-o", "name",
use_namespace: false, use_context: false, log_failure: false)
Expand Down
27 changes: 15 additions & 12 deletions test/integration/kubernetes_deploy_test.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,13 +254,18 @@ def test_multiple_invalid_k8s_specs_fails_on_apply_and_prints_template
assert_deploy_failure(result)
assert_logs_match_all([
"Command failed: apply -f",
"WARNING: Any resources not mentioned in the error below were likely created/updated.",
/Invalid templates: (Deployment|Service)-web.*\.yml, (Deployment|Service)-web.*\.yml/,
"Error from server (Invalid): error when creating",
"Error from server (Invalid): error when creating", # once for deployment, once for svc
"WARNING: Any resources not mentioned in the error(s) below were likely created/updated.",
/Invalid template: Deployment-web.*\.yml/,
"> Error message:",
/Error from server \(Invalid\): error when creating.*Deployment\.?\w* "web" is invalid/,
"> Template content:",
" name: http_test_is_really_long_and_invalid_chars",

/Invalid template: Service-web.*\.yml/,
"> Error message:",
/Error from server \(Invalid\): error when creating.*Service "web" is invalid/,
"> Template content:",
/(name|targetPort): http_test_is_really_long_and_invalid_chars/, # error in svc template
/(name|targetPort): http_test_is_really_long_and_invalid_chars/ # error in deployment template
" targetPort: http_test_is_really_long_and_invalid_chars"
], in_order: true)
end

Expand All @@ -272,9 +277,8 @@ def test_invalid_k8s_spec_that_is_valid_yaml_but_has_no_template_path_in_error_p
assert_deploy_failure(result)
assert_logs_match_all([
"Command failed: apply -f",
"WARNING: Any resources not mentioned in the error below were likely created/updated.",
"Invalid templates: See error message",
"> Error message:",
"WARNING: Any resources not mentioned in the error(s) below were likely created/updated.",
"Unidentified error(s):",
' The Service "web" is invalid:',
'spec.ports[0].targetPort: Invalid value: "http_test_is_really_long_and_invalid_chars"'
], in_order: true)
Expand Down Expand Up @@ -616,9 +620,8 @@ def test_output_when_switching_labels_incorrectly

assert_logs_match_all([
"Command failed: apply -f",
"WARNING: Any resources not mentioned in the error below were likely created/updated.",
"Invalid templates: See error message",
"> Error message:",
"WARNING: Any resources not mentioned in the error(s) below were likely created/updated.",
"Unidentified error(s):",
/The Deployment "web" is invalid.*`selector` does not match template `labels`/
], in_order: true)
end
Expand Down

0 comments on commit 2348c89

Please sign in to comment.