Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix postgres__regexp_instr not validating regex #249 #250

Merged
merged 6 commits into from
Mar 27, 2023
Merged

Fix postgres__regexp_instr not validating regex #249 #250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8762cd4
Add regex error tests
lookslikeitsnot Mar 25, 2023
4845030
Fix postgres__regexp_instr
lookslikeitsnot Mar 25, 2023
87bd8d3
cleaned up test regex
lookslikeitsnot Mar 25, 2023
58a5a8e
specified target adapters for regex flagged tests
lookslikeitsnot Mar 27, 2023
65ab547
Add regex config comments
clausherther Mar 27, 2023
3d39526
Merge branch 'bug/249_regexp_instr_always_valid' of github.com:looksl…
clausherther Mar 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 149 additions & 0 deletions integration_tests/models/schema_tests/schema.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,55 +4,204 @@ models:
columns:
- name: email_address
tests:
# match email address
- dbt_expectations.expect_column_values_to_match_regex:
regex: "@[^.]*"
# does not match email address, should fail
- dbt_expectations.expect_column_values_to_match_regex:
regex: "&[^.]*"
config:
error_if: "=0"
warn_if: "<4"
# match all uppercase, but match case-insensitive (where implemented)
- dbt_expectations.expect_column_values_to_match_regex:
regex: "[A-Z]"
flags: i
config:
enabled: "{{ target.type in ['postgres', 'snowflake', 'redshift' ] }}"
# match all uppercase, case-sensitive (where implemented), should fail
- dbt_expectations.expect_column_values_to_match_regex:
regex: "[A-Z]"
flags: c
config:
enabled: "{{ target.type in ['postgres', 'snowflake', 'redshift' ] }}"
error_if: "=0"
warn_if: "<4"
# do not match other non-email string, should pass
- dbt_expectations.expect_column_values_to_not_match_regex:
regex: "&[^.]*"
# match email address, should fail
- dbt_expectations.expect_column_values_to_not_match_regex:
regex: "@[^.]*"
config:
error_if: "=0"
warn_if: "<4"
# match all uppercase, case-sensitive (default), should pass
- dbt_expectations.expect_column_values_to_not_match_regex:
regex: "[A-Z]"
# match all lowercase, case-sensitive (default), should fail
- dbt_expectations.expect_column_values_to_not_match_regex:
regex: "[a-z]"
config:
error_if: "=0"
warn_if: "<4"
# do match one of email address or other non-email string
- dbt_expectations.expect_column_values_to_match_regex_list:
regex_list: ["@[^.]*", "&[^.]*"]
# do not match other non-email strings, should fail
- dbt_expectations.expect_column_values_to_match_regex_list:
regex_list: ["#[^.]*", "&[^.]*"]
config:
error_if: "=0"
warn_if: "<4"
# match all uppercase, but match case-insensitive (where implemented)
- dbt_expectations.expect_column_values_to_match_regex_list:
regex_list: ["[A-G]", "[H-Z]"]
flags: i
config:
enabled: "{{ target.type in ['postgres', 'snowflake', 'redshift' ] }}"
# match all uppercase, but match case-sensitive (where implemented), should fail
- dbt_expectations.expect_column_values_to_match_regex_list:
regex_list: ["[A-G]", "[H-Z]"]
flags: c
config:
enabled: "{{ target.type in ['postgres', 'snowflake', 'redshift' ] }}"
error_if: "=0"
warn_if: "<4"
# match email address or other string
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["@[^.]*", "&[^.]*"]
# match email address, should fail
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["@[^.]*", "@[^.]*"]
config:
error_if: "=0"
warn_if: "<4"
# do not match any of other non-email string
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["&[^.]*", "&[^.]*"]
match_on: all
# do not match any of email or other non-email string, should fail
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["@[^.]*", "&[^.]*"]
match_on: all
config:
error_if: "=0"
warn_if: "<4"
# do not match all uppercase
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["[A-G]", "[H-Z]"]
# do not match all uppercase or numbers, case-insensitive (where implemented)
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["[A-Z]", "[0-9]"]
flags: i
config:
enabled: "{{ target.type in ['postgres', 'snowflake', 'redshift' ] }}"
# do not match all uppercase and numbers, case-insensitive (where implemented)
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["[A-Z]", "[0-9]"]
flags: i
match_on: all
config:
enabled: "{{ target.type in ['postgres', 'snowflake', 'redshift' ] }}"
enabled: "{{ target.type in ['postgres', 'snowflake', 'redshift' ] }}"
error_if: "=0"
warn_if: "<4"
# match '@' anywhere in string
- dbt_expectations.expect_column_values_to_match_like_pattern:
like_pattern: "%@%"
# match '&' anywhere in string, should fail
- dbt_expectations.expect_column_values_to_match_like_pattern:
like_pattern: "%&%"
config:
error_if: "=0"
warn_if: "<4"
# do not match '&' anywhere in string
- dbt_expectations.expect_column_values_to_not_match_like_pattern:
like_pattern: "%&%"
# do not match '@' anywhere in string, should fail
- dbt_expectations.expect_column_values_to_not_match_like_pattern:
like_pattern: "%@%"
config:
error_if: "=0"
warn_if: "<4"
# match at least one of '@' or '&' anywhere in string
- dbt_expectations.expect_column_values_to_match_like_pattern_list:
like_pattern_list: ["%@%", "%&%"]
# match both '@' or '&' anywhere in string, should fail
- dbt_expectations.expect_column_values_to_match_like_pattern_list:
like_pattern_list: ["%@%", "%&%"]
match_on: all
config:
error_if: "=0"
warn_if: "<4"
# do not match at least one of '@' or '&' anywhere in string
- dbt_expectations.expect_column_values_to_not_match_like_pattern_list:
like_pattern_list: ["%@%", "%&%"]
# do not match either of '@' or '&' anywhere in string, should fail
- dbt_expectations.expect_column_values_to_not_match_like_pattern_list:
like_pattern_list: ["%@%", "%&%"]
match_on: all
config:
error_if: "=0"
warn_if: "<4"
- name: postal_code_5
tests:
- dbt_expectations.expect_column_values_to_match_regex:
regex: "^\\d{5}"
is_raw: True
- dbt_expectations.expect_column_values_to_match_regex:
regex: "^\\d{55}"
is_raw: True
config:
error_if: "=0"
warn_if: "<4"
- dbt_expectations.expect_column_values_to_not_match_regex:
regex: "@[^.]*"
is_raw: True
- dbt_expectations.expect_column_values_to_not_match_regex:
regex: "^\\d{5}"
is_raw: True
config:
error_if: "=0"
warn_if: "<4"
- dbt_expectations.expect_column_values_to_match_regex_list:
regex_list: ["^\\d{5}"]
is_raw: True
- dbt_expectations.expect_column_values_to_match_regex_list:
regex_list: ["^\\d{5}", "@[^.]*"]
is_raw: True
- dbt_expectations.expect_column_values_to_match_regex_list:
regex_list: ["^\\d{5}", "@[^.]*"]
is_raw: True
match_on: all
config:
error_if: "=0"
warn_if: "<4"
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["@[^.]*"]
is_raw: True
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["^\\d{5}", "@[^.]*"]
is_raw: True
- dbt_expectations.expect_column_values_to_not_match_regex_list:
regex_list: ["^\\d{5}", "@[^.]*"]
is_raw: True
match_on: all
config:
error_if: "=0"
warn_if: "<4"
- name: postal_code_5_3
tests:
- dbt_expectations.expect_column_values_to_match_regex:
regex: "^\\d{5}-\\d{3}"
is_raw: True
- dbt_expectations.expect_column_values_to_match_regex:
regex: "^\\d{5}-\\d{9}"
is_raw: True
config:
error_if: "=0"
warn_if: "<4"


- name: timeseries_data
Expand Down
2 changes: 1 addition & 1 deletion macros/regex/regexp_instr.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ regexp_instr({{ source_value }}, {{ regexp }}, {{ position }}, {{ occurrence }})
{# Postgres does not need to escape raw strings #}
{% macro postgres__regexp_instr(source_value, regexp, position, occurrence, is_raw, flags) %}
{% if flags %}{{ dbt_expectations._validate_flags(flags, 'bcegimnpqstwx') }}{% endif %}
array_length((select regexp_matches({{ source_value }}, '{{ regexp }}', '{{ flags }}')), 1)
coalesce(array_length((select regexp_matches({{ source_value }}, '{{ regexp }}', '{{ flags }}')), 1), 0)
{% endmacro %}

{# Unclear what Redshift does to escape raw strings #}
Expand Down