[resoto][chore] Fix require and bump dependencies (#1704)

requirements-all.txt

@@ -14,8 +14,8 @@ autocommand==2.2.2
 backoff==2.2.1
 bcrypt==4.0.1
 black==23.3.0
-boto3==1.26.161
-botocore==1.29.161
+boto3==1.26.165
+botocore==1.29.165
 brotli==1.0.9
 build==0.10.0
 cachetools==5.3.1
@@ -42,7 +42,7 @@ flake8==6.0.0
 frozendict==2.3.8
 frozenlist==1.3.3
 google-api-core==2.11.1
-google-api-python-client==2.90.0
+google-api-python-client==2.91.0
 google-auth==2.21.0
 google-auth-httplib2==0.1.0
 google-cloud-core==2.3.2
@@ -51,10 +51,10 @@ google-crc32c==1.5.0
 google-resumable-media==2.5.0
 googleapis-common-protos==1.59.1
 httplib2==0.22.0
-hypothesis==6.79.3
+hypothesis==6.80.0
 idna==3.4
 importlib-metadata==6.7.0
-inflect==6.0.4
+inflect==6.1.0
 iniconfig==2.0.0
 isort==5.12.0
 jaraco-collections==4.3.0
@@ -90,11 +90,11 @@ pathspec==0.11.1
 pep8-naming==0.13.3
 pint==0.22
 pip==23.1.2
-pip-tools==6.13.0
+pip-tools==6.14.0
 plantuml==0.3.0
 platformdirs==3.8.0
 pluggy==1.2.0
-portend==3.1.0
+portend==3.2.0
 posthog==3.0.1
 prometheus-client==0.17.0
 prompt-toolkit==3.0.38
@@ -108,7 +108,7 @@ pycares==4.3.0
 pycodestyle==2.10.0
 pycparser==2.21
 pycryptodomex==3.18.0
-pydantic==1.10.9
+pydantic==1.10.10
 pyflakes==3.0.1
 pygithub==1.59.0
 pygments==2.15.1
@@ -155,7 +155,7 @@ toolz==0.12.0
 tox==4.6.3
 transitions==0.9.0
 typeguard==4.0.0
-typing-extensions==4.6.3
+typing-extensions==4.7.1
 typish==1.9.3
 tzdata==2023.3
 tzlocal==5.0.1

requirements-extra.txt

@@ -12,8 +12,8 @@ attrs==23.1.0
 autocommand==2.2.2
 backoff==2.2.1
 bcrypt==4.0.1
-boto3==1.26.161
-botocore==1.29.161
+boto3==1.26.165
+botocore==1.29.165
 brotli==1.0.9
 cachetools==5.3.1
 cattrs==23.1.2
@@ -32,7 +32,7 @@ filelock==3.12.2
 frozendict==2.3.8
 frozenlist==1.3.3
 google-api-core==2.11.1
-google-api-python-client==2.90.0
+google-api-python-client==2.91.0
 google-auth==2.21.0
 google-auth-httplib2==0.1.0
 google-cloud-core==2.3.2
@@ -43,7 +43,7 @@ googleapis-common-protos==1.59.1
 httplib2==0.22.0
 idna==3.4
 importlib-metadata==6.7.0
-inflect==6.0.4
+inflect==6.1.0
 jaraco-collections==4.3.0
 jaraco-context==4.3.0
 jaraco-functools==3.8.0
@@ -71,7 +71,7 @@ paramiko==3.2.0
 parsy==2.1
 pint==0.22
 plantuml==0.3.0
-portend==3.1.0
+portend==3.2.0
 posthog==3.0.1
 prometheus-client==0.17.0
 prompt-toolkit==3.0.38
@@ -84,7 +84,7 @@ pyasn1-modules==0.3.0
 pycares==4.3.0
 pycparser==2.21
 pycryptodomex==3.18.0
-pydantic==1.10.9
+pydantic==1.10.10
 pygithub==1.59.0
 pygments==2.15.1
 pyjwt[crypto]==2.7.0
@@ -120,7 +120,7 @@ tenacity==8.2.2
 toolz==0.12.0
 transitions==0.9.0
 typeguard==4.0.0
-typing-extensions==4.6.3
+typing-extensions==4.7.1
 typish==1.9.3
 tzdata==2023.3
 tzlocal==5.0.1

requirements-test.txt

@@ -13,8 +13,8 @@ autocommand==2.2.2
 backoff==2.2.1
 bcrypt==4.0.1
 black==23.3.0
-boto3==1.26.161
-botocore==1.29.161
+boto3==1.26.165
+botocore==1.29.165
 brotli==1.0.9
 build==0.10.0
 cachetools==5.3.1
@@ -41,15 +41,15 @@ flake8==6.0.0
 frozendict==2.3.8
 frozenlist==1.3.3
 google-api-core==2.11.1
-google-api-python-client==2.90.0
+google-api-python-client==2.91.0
 google-auth==2.21.0
 google-auth-httplib2==0.1.0
 googleapis-common-protos==1.59.1
 httplib2==0.22.0
-hypothesis==6.79.3
+hypothesis==6.80.0
 idna==3.4
 importlib-metadata==6.7.0
-inflect==6.0.4
+inflect==6.1.0
 iniconfig==2.0.0
 isort==5.12.0
 jaraco-collections==4.3.0
@@ -83,11 +83,11 @@ pathspec==0.11.1
 pep8-naming==0.13.3
 pint==0.22
 pip==23.1.2
-pip-tools==6.13.0
+pip-tools==6.14.0
 plantuml==0.3.0
 platformdirs==3.8.0
 pluggy==1.2.0
-portend==3.1.0
+portend==3.2.0
 posthog==3.0.1
 prometheus-client==0.17.0
 prompt-toolkit==3.0.38
@@ -98,7 +98,7 @@ pyasn1-modules==0.3.0
 pycares==4.3.0
 pycodestyle==2.10.0
 pycparser==2.21
-pydantic==1.10.9
+pydantic==1.10.10
 pyflakes==3.0.1
 pygithub==1.59.0
 pygments==2.15.1
@@ -141,7 +141,7 @@ toolz==0.12.0
 tox==4.6.3
 transitions==0.9.0
 typeguard==4.0.0
-typing-extensions==4.6.3
+typing-extensions==4.7.1
 typish==1.9.3
 tzdata==2023.3
 tzlocal==5.0.1

requirements.txt

@@ -11,8 +11,8 @@ attrs==23.1.0
 autocommand==2.2.2
 backoff==2.2.1
 bcrypt==4.0.1
-boto3==1.26.161
-botocore==1.29.161
+boto3==1.26.165
+botocore==1.29.165
 brotli==1.0.9
 cachetools==5.3.1
 cattrs==23.1.2
@@ -30,14 +30,14 @@ fastjsonschema==2.16.3
 frozendict==2.3.8
 frozenlist==1.3.3
 google-api-core==2.11.1
-google-api-python-client==2.90.0
+google-api-python-client==2.91.0
 google-auth==2.21.0
 google-auth-httplib2==0.1.0
 googleapis-common-protos==1.59.1
 httplib2==0.22.0
 idna==3.4
 importlib-metadata==6.7.0
-inflect==6.0.4
+inflect==6.1.0
 jaraco-collections==4.3.0
 jaraco-context==4.3.0
 jaraco-functools==3.8.0
@@ -62,7 +62,7 @@ paramiko==3.2.0
 parsy==2.1
 pint==0.22
 plantuml==0.3.0
-portend==3.1.0
+portend==3.2.0
 posthog==3.0.1
 prometheus-client==0.17.0
 prompt-toolkit==3.0.38
@@ -72,7 +72,7 @@ pyasn1==0.5.0
 pyasn1-modules==0.3.0
 pycares==4.3.0
 pycparser==2.21
-pydantic==1.10.9
+pydantic==1.10.10
 pygithub==1.59.0
 pygments==2.15.1
 pyjwt[crypto]==2.7.0
@@ -103,7 +103,7 @@ tenacity==8.2.2
 toolz==0.12.0
 transitions==0.9.0
 typeguard==4.0.0
-typing-extensions==4.6.3
+typing-extensions==4.7.1
 typish==1.9.3
 tzdata==2023.3
 tzlocal==5.0.1

tools/requirements.py

@@ -60,13 +60,30 @@ def filter_dependencies(deps: List[str]) -> List[str]:
     return [dep for dep in deps if not any(name in dep for name in filter_out)]
 
 
-def compile_dependencies(name: Optional[str], deps: List[str]) -> None:
+def compile_dependencies(name: Optional[str], deps: List[str], all_dependencies: List[str]) -> List[str]:
+    # If a dependency is defined in all, use that version
+    def dependency_line(dep: str) -> str:
+        d_name, d_version = dep.split("==", maxsplit=1)
+        if defined_in_all := [a for a in all_dependencies if a.startswith(d_name + "==")]:
+            assert len(defined_in_all) == 1, f"More than one dependency with this name: {d_name}"
+            return defined_in_all[0]
+        else:
+            return dep
+
+    print(f"Compile dependencies for {name or 'prod'}")
     delim = "-" + name if name else ""
     with open(f"requirements{delim}-in.txt", "w") as f:
         f.write("\n".join(deps))
     args = "-q --no-annotate --resolver=backtracking --upgrade --allow-unsafe --no-header  --unsafe-package n/a"
     os.system(f"pip-compile {args} --output-file requirements{delim}.txt requirements{delim}-in.txt")
     os.remove(f"requirements{delim}-in.txt")
+    # make sure, none of the filtered dependencies was selected as transitive dependency
+    with open(f"requirements{delim}.txt", "r+") as f:
+        lines = [dependency_line(line) for line in f.readlines() if not any(name in line for name in filter_out)]
+        f.seek(0)
+        f.writelines(lines)
+        f.truncate()
+        return lines
 
 
 def combine_dependencies() -> None:
@@ -76,13 +93,16 @@ def combine_dependencies() -> None:
         prod_dependencies.extend(filter_dependencies(project.dependencies))
         for name, deps in project.optional_dependencies.items():
             optional_dependencies[name].extend(filter_dependencies(deps))
-    all_dependencies = prod_dependencies.copy()
 
-    compile_dependencies(None, prod_dependencies)
+    # gather all dependencies
+    all_dependencies = prod_dependencies.copy()
     for name, deps in optional_dependencies.items():
         all_dependencies.extend(deps)
-        compile_dependencies(name, deps + prod_dependencies)
-    compile_dependencies("all", all_dependencies)
+    all_compiled = compile_dependencies("all", all_dependencies, [])
+
+    compile_dependencies(None, prod_dependencies, all_compiled)
+    for name, deps in optional_dependencies.items():
+        compile_dependencies(name, deps + prod_dependencies, all_compiled)
 
 
 if __name__ == "__main__":