Skip to content

Commit

Permalink
Refactor: Simplify a few loops (#33736)
Browse files Browse the repository at this point in the history
  • Loading branch information
eumiro committed Aug 26, 2023
1 parent 272b40a commit 784e0ef
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 17 deletions.
8 changes: 4 additions & 4 deletions airflow/providers/apache/hive/transfers/s3_to_hive.py
Expand Up @@ -247,16 +247,16 @@ def _match_headers(self, header_list):
"Headers count mismatch File headers:\n %s\nField names: \n %s\n", header_list, field_names
)
return False
test_field_match = [h1.lower() == h2.lower() for h1, h2 in zip(header_list, field_names)]
if not all(test_field_match):
test_field_match = all(h1.lower() == h2.lower() for h1, h2 in zip(header_list, field_names))
if test_field_match:
return True
else:
self.log.warning(
"Headers do not match field names File headers:\n %s\nField names: \n %s\n",
header_list,
field_names,
)
return False
else:
return True

@staticmethod
def _delete_top_row_and_compress(input_file_name, output_file_ext, dest_dir):
Expand Down
12 changes: 6 additions & 6 deletions airflow/providers/google/cloud/hooks/bigquery.py
Expand Up @@ -3154,15 +3154,15 @@ def get_records(self, query_results: dict[str, Any], as_dict: bool = False) -> l
if "rows" in query_results and query_results["rows"]:
rows = query_results["rows"]
fields = query_results["schema"]["fields"]
fields_names = [field["name"] for field in fields]
col_types = [field["type"] for field in fields]
for dict_row in rows:
typed_row = [bq_cast(vs["v"], col_types[idx]) for idx, vs in enumerate(dict_row["f"])]
if not as_dict:
buffer.append(typed_row)
else:
fields_names = [field["name"] for field in fields]
typed_row_dict = {k: v for k, v in zip(fields_names, typed_row)}
typed_row = [bq_cast(vs["v"], col_type) for vs, col_type in zip(dict_row["f"], col_types)]
if as_dict:
typed_row_dict = dict(zip(fields_names, typed_row))
buffer.append(typed_row_dict)
else:
buffer.append(typed_row)
return buffer

def value_check(
Expand Down
8 changes: 3 additions & 5 deletions scripts/ci/pre_commit/pre_commit_replace_bad_characters.py
Expand Up @@ -56,13 +56,11 @@ def main() -> int:
count_changes = 0
path = Path(file_string)
text = path.read_text()
for index in range(len(matches)):
current_match = matches[index]
text, new_count_changes = current_match.subn(REPLACEMENTS[index].replacement, text)
for match, spec in zip(matches, REPLACEMENTS):
text, new_count_changes = match.subn(spec.replacement, text)
if new_count_changes:
console.print(
f"[yellow] Performed {new_count_changes} replacements "
f"of {REPLACEMENTS[index].description}[/]: {path}"
f"[yellow] Performed {new_count_changes} replacements of {spec.description}[/]: {path}"
)
count_changes += new_count_changes
if count_changes:
Expand Down
3 changes: 1 addition & 2 deletions tests/test_utils/mock_executor.py
Expand Up @@ -69,8 +69,7 @@ def sort_by(item):

open_slots = self.parallelism - len(self.running)
sorted_queue = sorted(self.queued_tasks.items(), key=sort_by)
for index in range(min((open_slots, len(sorted_queue)))):
(key, (_, _, _, ti)) = sorted_queue[index]
for key, (_, _, _, ti) in sorted_queue[:open_slots]:
self.queued_tasks.pop(key)
ti._try_number += 1
state = self.mock_task_results[key]
Expand Down

0 comments on commit 784e0ef

Please sign in to comment.