Skip to content

Commit

Permalink
fix: correct trimming of state backend URIs for filesystem backends (m…
Browse files Browse the repository at this point in the history
…eltano#7320)

* Correct misuse of `rstrip` in favor of new 3.8-compatible `remove_suffix`

* Correct version check
  • Loading branch information
cjohnhanson committed Feb 15, 2023
1 parent c5b2f88 commit 5d04802
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 4 deletions.
11 changes: 7 additions & 4 deletions src/meltano/core/state_store/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

from meltano.core.job_state import JobState
from meltano.core.state_store.base import StateStoreManager
from meltano.core.utils import remove_suffix

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -87,12 +88,14 @@ def get_reader(self, path: str) -> Iterator[TextIOWrapper]:
"""
if self.client:
with open(
self.join_path(self.uri.rstrip(self.state_dir), path),
self.join_path(remove_suffix(self.uri, (self.state_dir)), path),
transport_params={"client": self.client},
) as reader:
yield reader
else:
with open(self.join_path(self.uri.rstrip(self.state_dir), path)) as reader:
with open(
self.join_path(remove_suffix(self.uri, self.state_dir), path)
) as reader:
yield reader

@contextmanager
Expand All @@ -107,14 +110,14 @@ def get_writer(self, path: str) -> Iterator[TextIOWrapper]:
"""
try:
with open(
self.join_path(self.uri.rstrip(self.state_dir), path),
self.join_path(remove_suffix(self.uri, self.state_dir), path),
"w+",
transport_params={"client": self.client} if self.client else {},
) as writer:
yield writer
except NotImplementedError:
with open(
self.join_path(self.uri.rstrip(self.state_dir), path),
self.join_path(remove_suffix(self.uri, self.state_dir), path),
"w",
transport_params={"client": self.client} if self.client else {},
) as writer:
Expand Down
19 changes: 19 additions & 0 deletions src/meltano/core/utils/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -811,3 +811,22 @@ def _deep_merge(a, b, strategies):
):
break
return base


def remove_suffix(string: str, suffix: str) -> str:
"""Remove suffix from string.
Compatible with Python 3.8
Args:
string: the string to remove suffix from
suffix: the suffix to remove
Returns:
The changed string
"""
if sys.version_info >= (3, 9):
return string.removesuffix(suffix)
elif string.endswith(suffix):
return string[: -len(suffix)]
return string
7 changes: 7 additions & 0 deletions tests/meltano/core/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
flatten,
nest,
pop_at_path,
remove_suffix,
set_at_path,
)

Expand Down Expand Up @@ -155,3 +156,9 @@ def test_expand_env_vars_nested():
}

assert expand_env_vars(input_dict, env) == expected_output


def test_remove_suffix():
assert remove_suffix("a_string", "ing") == "a_str"
assert remove_suffix("a_string", "in") == "a_string"
assert remove_suffix("a_string", "gni") == "a_string"

0 comments on commit 5d04802

Please sign in to comment.