From 96297b864f028685bfae2a64c5c4dcb968c00432 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Tue, 7 May 2024 07:03:31 +0530 Subject: [PATCH 01/22] Update pyproject.toml: to add dynamic optional dependencies --- pyproject.toml | 40 ++++------------------------------------ requirements-base.txt | 20 ++++++++++++++++++++ requirements.txt | 29 +++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 36 deletions(-) create mode 100644 requirements-base.txt create mode 100644 requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 81581d48..cf441464 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,29 +20,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] requires-python = ">=3.9" -dependencies = [ - "aiofiles", - "emoji", - "fastapi", - "httpx", - "importlib_metadata>=4.6; python_version<'3.10'", - "packaging", - "panel==1.3.8", - "pydantic>=2", - "pydantic-core", - "pydantic-settings>=2", - "PyJWT", - "python-multipart", - "redis", - "questionary", - "rich", - "sqlalchemy>=2", - "starlette", - "tomlkit", - "typer", - "uvicorn", -] -dynamic = ["version"] +dynamic = ["dependencies", "optional-dependencies", "version"] [project.urls] Homepage = "https://ragna.chat" @@ -50,19 +28,9 @@ Documentation = "https://ragna.chat" Changelog = "https://ragna.chat/en/stable/references/release-notes/" Repository = "https://github.com/Quansight/ragna" -[project.optional-dependencies] -# to update the array below, run scripts/update_optional_dependencies.py -all = [ - "chromadb>=0.4.13", - "httpx_sse", - "ijson", - "lancedb>=0.2", - "pyarrow", - "pymupdf>=1.23.6", - "python-docx", - "python-pptx", - "tiktoken", -] +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} +optional-dependencies = {base = { file = ["requirements-base.txt"] }} [tool.setuptools_scm] write_to = "ragna/_version.py" diff --git a/requirements-base.txt b/requirements-base.txt new file mode 100644 index 00000000..13eb3e66 --- /dev/null +++ b/requirements-base.txt @@ -0,0 +1,20 @@ +aiofiles +emoji +fastapi +httpx +importlib_metadata>=4.6; python_version<'3.10' +packaging +panel==1.3.8 +pydantic>=2 +pydantic-core +pydantic-settings>=2 +PyJWT +python-multipart +redis +questionary +rich +sqlalchemy>=2 +starlette +tomlkit +typer +uvicorn diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..7da3dd6f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,29 @@ +aiofiles +emoji +fastapi +httpx +importlib_metadata>=4.6; python_version<'3.10' +packaging +panel==1.3.8 +pydantic>=2 +pydantic-core +pydantic-settings>=2 +PyJWT +python-multipart +redis +questionary +rich +sqlalchemy>=2 +starlette +tomlkit +typer +uvicorn +chromadb>=0.4.13 +httpx_sse +ijson +lancedb>=0.2 +pyarrow +pymupdf>=1.23.6 +python-docx +python-pptx +tiktoken From f9b8c393cee3e878eaa4299988d09fc4182e506a Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Sat, 11 May 2024 23:40:41 +0530 Subject: [PATCH 02/22] Add concurrent `setup.py` with almost same configurations as `pyproject.toml` --- setup.py | 117 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..ce5faf3b --- /dev/null +++ b/setup.py @@ -0,0 +1,117 @@ +from setuptools import find_packages, setup + +# Read requirements from requirements.txt +with open("requirements-base.txt") as f: + install_requires = f.read().splitlines() + print(f) + +setup( + name="ragna-base", + description="RAG orchestration framework", + license="BSD 3-Clause License", + author="Ragna Development Team", + author_email="connect@quansight.com", + url="https://ragna.chat", + classifiers=[ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + ], + python_requires=">=3.9", + packages=find_packages(), + scripts=["ragna/__main__.py"], + install_requires=install_requires, # Use dependencies from requirements.txt + setup_requires=[ + "setuptools>=45", + "setuptools_scm[toml]>=6.2", + ], + extras_require={ + "optional": [], + }, + package_data={}, + include_package_data=True, + entry_points={}, + # Ruff configuration + ruff={ + "lint": { + "select": [ + "E", + "F", + # import sorting + "I001", + ], + # Ignore line too long, because due to black, the error can only occur for strings + "ignore": ["E501"], + "per-file-ignores": { + # ignore unused imports and imports not at the top of the file in __init__.py files + "__init__.py": ["F401", "E402"], + # The examples often have imports below the top of the file to follow the narrative + "docs/examples/**/*.py": ["E402", "F704", "I001"], + "docs/tutorials/**/*.py": ["E402", "F704", "I001"], + }, + } + }, + # Pytest configuration + pytest={ + "ini_options": { + "minversion": "6.0", + "addopts": "-ra --tb=short --asyncio-mode=auto", + "testpaths": ["tests"], + "filterwarnings": [ + "error", + "ignore::ResourceWarning", + # httpx 0.27.0 deprecated some functionality that the test client of starlette / + # FastApi use. This should be resolved by the next release of these libraries. + # See https://github.com/encode/starlette/issues/2524 + "ignore:The 'app' shortcut is now deprecated:DeprecationWarning", + ], + "xfail_strict": True, + } + }, + # Mypy configuration + mypy={ + "files": "ragna", + "plugins": ["sqlmypy"], + "show_error_codes": True, + "pretty": True, + "disallow_untyped_calls": True, + "disallow_untyped_defs": True, + "disallow_incomplete_defs": True, + "allow_redefinition": True, + "no_implicit_optional": True, + "warn_redundant_casts": True, + "warn_unused_ignores": True, + "warn_return_any": True, + "warn_unused_configs": True, + "overrides": [ + { + "module": ["ragna.deploy._ui.*"], + "disallow_untyped_calls": False, + "disallow_untyped_defs": False, + "disallow_incomplete_defs": False, + }, + { + "module": [ + "docx", + "fitz", + "ijson", + "lancedb", + "param", + "pptx", + "pyarrow", + "sentence_transformers", + ], + "ignore_missing_imports": True, + }, + { + "module": ["ragna.deploy._api.orm"], + "disable_error_code": ["var-annotated"], + }, + { + "module": ["ragna.source_storages.*", "ragna.assistants.*"], + "disable_error_code": ["override"], + }, + ], + }, +) From 65753f168bea772c842e3bc49e1d83a8f9dd0193 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Tue, 14 May 2024 12:41:04 +0530 Subject: [PATCH 03/22] Use multiple pyproject in `scripts/` and wrap it inside nox to build while symlinking with any of them dynamically --- noxfile.py | 27 +++++ scripts/pyproject-base.toml | 144 +++++++++++++++++++++++ pyproject.toml => scripts/pyproject.toml | 1 - 3 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 noxfile.py create mode 100644 scripts/pyproject-base.toml rename pyproject.toml => scripts/pyproject.toml (97%) diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 00000000..b6ee384a --- /dev/null +++ b/noxfile.py @@ -0,0 +1,27 @@ +import os + +import nox + + +@nox.session(name="build") +def build(session): + setup_script = "scripts/pyproject.toml" + symlink_pyproject(setup_script) + session.install("build") + session.run("python", "-m", "build") + + +@nox.session(name="build-base") +def build_bar(session): + setup_script = "scripts/pyproject-base.toml" + symlink_pyproject(setup_script) + session.install("build") + session.run("python", "-m", "build") + + +def symlink_pyproject(script): + if os.path.exists(script): + # Remove existing pyproject.toml if it exists + if os.path.exists("pyproject.toml"): + os.remove("pyproject.toml") + os.symlink(script, "pyproject.toml") diff --git a/scripts/pyproject-base.toml b/scripts/pyproject-base.toml new file mode 100644 index 00000000..9e7333d9 --- /dev/null +++ b/scripts/pyproject-base.toml @@ -0,0 +1,144 @@ +[build-system] +requires = [ + "setuptools>=45", + "setuptools_scm[toml]>=6.2", +] +build-backend = "setuptools.build_meta" + +[project] +name = "ragna-base" +description = "RAG orchestration framework" +license = {file = "LICENSE"} +authors = [ + { name = "Ragna Development Team", email = "connect@quansight.com" }, +] +readme = "README.md" +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +requires-python = ">=3.9" +dynamic = ["dependencies", "optional-dependencies", "version"] + +[project.urls] +Homepage = "https://ragna.chat" +Documentation = "https://ragna.chat" +Changelog = "https://ragna.chat/en/stable/references/release-notes/" +Repository = "https://github.com/Quansight/ragna" + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements-base.txt"]} + +[tool.setuptools_scm] +write_to = "ragna/_version.py" +version_scheme = "release-branch-semver" +local_scheme = "node-and-timestamp" + +[project.scripts] +ragna = "ragna.__main__:app" + +[tool.setuptools.packages.find] +include = [ + "ragna*", +] + +[tool.ruff.lint] +select = [ + "E", + "F", + # import sorting + "I001" +] +# Ignore line too long, because due to black, the error can only occur for strings +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +# ignore unused imports and imports not at the top of the file in __init__.py files +"__init__.py" = ["F401", "E402"] +# The examples often have imports below the top of the file to follow the narrative +"docs/examples/**/*.py" = ["E402", "F704", "I001"] +"docs/tutorials/**/*.py" = ["E402", "F704", "I001"] + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra --tb=short --asyncio-mode=auto" +testpaths = [ + "tests", +] +filterwarnings = [ + "error", + "ignore::ResourceWarning", + # httpx 0.27.0 deprecated some functionality that the test client of starlette / + # FastApi use. This should be resolved by the next release of these libraries. + # See https://github.com/encode/starlette/issues/2524 + "ignore:The 'app' shortcut is now deprecated:DeprecationWarning", +] +xfail_strict = true + +[tool.mypy] + +files = "ragna" + +plugins = [ + "sqlmypy", +] + +show_error_codes = true +pretty = true + +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true + +allow_redefinition = true +no_implicit_optional = true + +warn_redundant_casts = true +warn_unused_ignores = true + +warn_return_any = true +warn_unused_configs = true + +[[tool.mypy.overrides]] +module = [ + "ragna.deploy._ui.*", +] +disallow_untyped_calls = false +disallow_untyped_defs = false +disallow_incomplete_defs = false + +[[tool.mypy.overrides]] +module = [ + "docx", + "fitz", + "ijson", + "lancedb", + "param", + "pptx", + "pyarrow", + "sentence_transformers", +] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = [ + "ragna.deploy._api.orm", +] +# Our ORM schema doesn't really work with mypy. There are some other ways to define it +# to play ball. We should do that in the future. +disable_error_code = [ + "var-annotated", +] + +[[tool.mypy.overrides]] +# It is a fundamental feature of the components to request more parameters than the base +# class. Thus, we just silence mypy here. +module = [ + "ragna.source_storages.*", + "ragna.assistants.*" +] +disable_error_code = [ + "override", +] diff --git a/pyproject.toml b/scripts/pyproject.toml similarity index 97% rename from pyproject.toml rename to scripts/pyproject.toml index 809beae6..f606384f 100644 --- a/pyproject.toml +++ b/scripts/pyproject.toml @@ -30,7 +30,6 @@ Repository = "https://github.com/Quansight/ragna" [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} -optional-dependencies = {base = { file = ["requirements-base.txt"] }} [tool.setuptools_scm] write_to = "ragna/_version.py" From 3201c1fc0832ac5559ededb74946803097943a84 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Tue, 14 May 2024 15:42:36 +0530 Subject: [PATCH 04/22] Also add pyproject in root for CI build and testing --- pyproject.toml | 144 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..f606384f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,144 @@ +[build-system] +requires = [ + "setuptools>=45", + "setuptools_scm[toml]>=6.2", +] +build-backend = "setuptools.build_meta" + +[project] +name = "Ragna" +description = "RAG orchestration framework" +license = {file = "LICENSE"} +authors = [ + { name = "Ragna Development Team", email = "connect@quansight.com" }, +] +readme = "README.md" +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +requires-python = ">=3.9" +dynamic = ["dependencies", "optional-dependencies", "version"] + +[project.urls] +Homepage = "https://ragna.chat" +Documentation = "https://ragna.chat" +Changelog = "https://ragna.chat/en/stable/references/release-notes/" +Repository = "https://github.com/Quansight/ragna" + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} + +[tool.setuptools_scm] +write_to = "ragna/_version.py" +version_scheme = "release-branch-semver" +local_scheme = "node-and-timestamp" + +[project.scripts] +ragna = "ragna.__main__:app" + +[tool.setuptools.packages.find] +include = [ + "ragna*", +] + +[tool.ruff.lint] +select = [ + "E", + "F", + # import sorting + "I001" +] +# Ignore line too long, because due to black, the error can only occur for strings +ignore = ["E501"] + +[tool.ruff.lint.per-file-ignores] +# ignore unused imports and imports not at the top of the file in __init__.py files +"__init__.py" = ["F401", "E402"] +# The examples often have imports below the top of the file to follow the narrative +"docs/examples/**/*.py" = ["E402", "F704", "I001"] +"docs/tutorials/**/*.py" = ["E402", "F704", "I001"] + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = "-ra --tb=short --asyncio-mode=auto" +testpaths = [ + "tests", +] +filterwarnings = [ + "error", + "ignore::ResourceWarning", + # httpx 0.27.0 deprecated some functionality that the test client of starlette / + # FastApi use. This should be resolved by the next release of these libraries. + # See https://github.com/encode/starlette/issues/2524 + "ignore:The 'app' shortcut is now deprecated:DeprecationWarning", +] +xfail_strict = true + +[tool.mypy] + +files = "ragna" + +plugins = [ + "sqlmypy", +] + +show_error_codes = true +pretty = true + +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_incomplete_defs = true + +allow_redefinition = true +no_implicit_optional = true + +warn_redundant_casts = true +warn_unused_ignores = true + +warn_return_any = true +warn_unused_configs = true + +[[tool.mypy.overrides]] +module = [ + "ragna.deploy._ui.*", +] +disallow_untyped_calls = false +disallow_untyped_defs = false +disallow_incomplete_defs = false + +[[tool.mypy.overrides]] +module = [ + "docx", + "fitz", + "ijson", + "lancedb", + "param", + "pptx", + "pyarrow", + "sentence_transformers", +] +ignore_missing_imports = true + +[[tool.mypy.overrides]] +module = [ + "ragna.deploy._api.orm", +] +# Our ORM schema doesn't really work with mypy. There are some other ways to define it +# to play ball. We should do that in the future. +disable_error_code = [ + "var-annotated", +] + +[[tool.mypy.overrides]] +# It is a fundamental feature of the components to request more parameters than the base +# class. Thus, we just silence mypy here. +module = [ + "ragna.source_storages.*", + "ragna.assistants.*" +] +disable_error_code = [ + "override", +] From 551924d2ec92da707c52e12cd9cbf800fee8b12b Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Tue, 14 May 2024 15:50:23 +0530 Subject: [PATCH 05/22] Ignore semver warning in pyproject (this change was overlooked while merging) --- pyproject.toml | 6 ++---- scripts/pyproject-base.toml | 6 ++---- scripts/pyproject.toml | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f606384f..a6f1fbb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,10 +70,8 @@ testpaths = [ filterwarnings = [ "error", "ignore::ResourceWarning", - # httpx 0.27.0 deprecated some functionality that the test client of starlette / - # FastApi use. This should be resolved by the next release of these libraries. - # See https://github.com/encode/starlette/issues/2524 - "ignore:The 'app' shortcut is now deprecated:DeprecationWarning", + # https://github.com/lancedb/lancedb/issues/1296 + "ignore:Function 'semver.parse_version_info' is deprecated:DeprecationWarning" ] xfail_strict = true diff --git a/scripts/pyproject-base.toml b/scripts/pyproject-base.toml index 9e7333d9..8cc5f9c3 100644 --- a/scripts/pyproject-base.toml +++ b/scripts/pyproject-base.toml @@ -70,10 +70,8 @@ testpaths = [ filterwarnings = [ "error", "ignore::ResourceWarning", - # httpx 0.27.0 deprecated some functionality that the test client of starlette / - # FastApi use. This should be resolved by the next release of these libraries. - # See https://github.com/encode/starlette/issues/2524 - "ignore:The 'app' shortcut is now deprecated:DeprecationWarning", + # https://github.com/lancedb/lancedb/issues/1296 + "ignore:Function 'semver.parse_version_info' is deprecated:DeprecationWarning" ] xfail_strict = true diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml index f606384f..a6f1fbb2 100644 --- a/scripts/pyproject.toml +++ b/scripts/pyproject.toml @@ -70,10 +70,8 @@ testpaths = [ filterwarnings = [ "error", "ignore::ResourceWarning", - # httpx 0.27.0 deprecated some functionality that the test client of starlette / - # FastApi use. This should be resolved by the next release of these libraries. - # See https://github.com/encode/starlette/issues/2524 - "ignore:The 'app' shortcut is now deprecated:DeprecationWarning", + # https://github.com/lancedb/lancedb/issues/1296 + "ignore:Function 'semver.parse_version_info' is deprecated:DeprecationWarning" ] xfail_strict = true From a1a4f4f92603244ede751861e0bcf1c22572e197 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Fri, 17 May 2024 13:21:50 +0530 Subject: [PATCH 06/22] Use `setup.py` as shim --- setup.py | 126 +++++++------------------------------------------------ 1 file changed, 14 insertions(+), 112 deletions(-) diff --git a/setup.py b/setup.py index ce5faf3b..289c196f 100644 --- a/setup.py +++ b/setup.py @@ -1,117 +1,19 @@ -from setuptools import find_packages, setup +import os + +from setuptools import setup + +with open("requirements.txt") as f: + ragna_dependencies = f.read().splitlines() -# Read requirements from requirements.txt with open("requirements-base.txt") as f: - install_requires = f.read().splitlines() - print(f) + base_dependencies = f.read().splitlines() + +name = "ragna-base" if os.environ.get("BUILD_RAGNA_BASE") else "ragna" +dependencies = ( + base_dependencies if os.environ.get("BUILD_RAGNA_BASE") else ragna_dependencies +) setup( - name="ragna-base", - description="RAG orchestration framework", - license="BSD 3-Clause License", - author="Ragna Development Team", - author_email="connect@quansight.com", - url="https://ragna.chat", - classifiers=[ - "Development Status :: 4 - Beta", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - ], - python_requires=">=3.9", - packages=find_packages(), - scripts=["ragna/__main__.py"], - install_requires=install_requires, # Use dependencies from requirements.txt - setup_requires=[ - "setuptools>=45", - "setuptools_scm[toml]>=6.2", - ], - extras_require={ - "optional": [], - }, - package_data={}, - include_package_data=True, - entry_points={}, - # Ruff configuration - ruff={ - "lint": { - "select": [ - "E", - "F", - # import sorting - "I001", - ], - # Ignore line too long, because due to black, the error can only occur for strings - "ignore": ["E501"], - "per-file-ignores": { - # ignore unused imports and imports not at the top of the file in __init__.py files - "__init__.py": ["F401", "E402"], - # The examples often have imports below the top of the file to follow the narrative - "docs/examples/**/*.py": ["E402", "F704", "I001"], - "docs/tutorials/**/*.py": ["E402", "F704", "I001"], - }, - } - }, - # Pytest configuration - pytest={ - "ini_options": { - "minversion": "6.0", - "addopts": "-ra --tb=short --asyncio-mode=auto", - "testpaths": ["tests"], - "filterwarnings": [ - "error", - "ignore::ResourceWarning", - # httpx 0.27.0 deprecated some functionality that the test client of starlette / - # FastApi use. This should be resolved by the next release of these libraries. - # See https://github.com/encode/starlette/issues/2524 - "ignore:The 'app' shortcut is now deprecated:DeprecationWarning", - ], - "xfail_strict": True, - } - }, - # Mypy configuration - mypy={ - "files": "ragna", - "plugins": ["sqlmypy"], - "show_error_codes": True, - "pretty": True, - "disallow_untyped_calls": True, - "disallow_untyped_defs": True, - "disallow_incomplete_defs": True, - "allow_redefinition": True, - "no_implicit_optional": True, - "warn_redundant_casts": True, - "warn_unused_ignores": True, - "warn_return_any": True, - "warn_unused_configs": True, - "overrides": [ - { - "module": ["ragna.deploy._ui.*"], - "disallow_untyped_calls": False, - "disallow_untyped_defs": False, - "disallow_incomplete_defs": False, - }, - { - "module": [ - "docx", - "fitz", - "ijson", - "lancedb", - "param", - "pptx", - "pyarrow", - "sentence_transformers", - ], - "ignore_missing_imports": True, - }, - { - "module": ["ragna.deploy._api.orm"], - "disable_error_code": ["var-annotated"], - }, - { - "module": ["ragna.source_storages.*", "ragna.assistants.*"], - "disable_error_code": ["override"], - }, - ], - }, + name=name, + install_requires=dependencies, ) From 12ac91fc6f33d0beee5a5f9e7f854271edf4af2d Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 22 May 2024 15:15:19 +0530 Subject: [PATCH 07/22] Add some helping logging points in `setup` --- setup.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/setup.py b/setup.py index 289c196f..6861a740 100644 --- a/setup.py +++ b/setup.py @@ -2,17 +2,26 @@ from setuptools import setup +print("BUILD_RAGNA_BASE=", os.environ["BUILD_RAGNA_BASE"]) + with open("requirements.txt") as f: ragna_dependencies = f.read().splitlines() with open("requirements-base.txt") as f: base_dependencies = f.read().splitlines() +is_base_build = os.environ.get("BUILD_RAGNA_BASE") + name = "ragna-base" if os.environ.get("BUILD_RAGNA_BASE") else "ragna" dependencies = ( base_dependencies if os.environ.get("BUILD_RAGNA_BASE") else ragna_dependencies ) +print(f"Building package: {name}") +print( + f"Using dependencies from {'requirements-base.txt' if is_base_build else 'requirements.txt'}" +) + setup( name=name, install_requires=dependencies, From ca5f58b32d134825073a0d033c43560edf619e35 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 22 May 2024 17:54:23 +0530 Subject: [PATCH 08/22] Add basic workflow to test build --- .github/workflows/build_package.yml | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/build_package.yml diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml new file mode 100644 index 00000000..d5a2ab28 --- /dev/null +++ b/.github/workflows/build_package.yml @@ -0,0 +1,46 @@ +name: Publish + +on: + release: + types: + - created + +jobs: + publish: + runs-on: ubuntu-latest + strategy: + matrix: + package: + - ragna + - ragna-base + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install build dependencies + run: pip install build + + - name: Set BUILD_RAGNA_BASE environment variable + run: | + if [ "${{ matrix.package }}" == "ragna-base" ]; then + echo "BUILD_RAGNA_BASE=1" >> $GITHUB_ENV + else + echo "BUILD_RAGNA_BASE=" >> $GITHUB_ENV + fi + + - name: Build distribution + run: | + echo "Building package: ${{ matrix.package }}" + python -m build + + - name: Verify package + run: | + echo "Verifying package: ${{ matrix.package }}" + pip install dist/*.whl + pip check From 43c2ec88c5739c17b32bae0527b7a346b6647d02 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 22 May 2024 17:55:17 +0530 Subject: [PATCH 09/22] Use pipdeptree to visualise dependency tree of built package --- .github/workflows/build_package.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index d5a2ab28..ebd52278 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -44,3 +44,8 @@ jobs: echo "Verifying package: ${{ matrix.package }}" pip install dist/*.whl pip check + + - name: Visualize dependency tree of built package + run: | + pip install pipdeptree + pipdeptree -d 1 From 6c8f6147bdd0ec473f280cc1b32481e950624ccd Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 22 May 2024 18:04:21 +0530 Subject: [PATCH 10/22] Temporarily run build workflow on PRs --- .github/workflows/build_package.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index ebd52278..9c91cd1e 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -1,9 +1,10 @@ name: Publish on: - release: - types: - - created + pull_request: + push: + branches: + - release/* jobs: publish: From 05411b740a57395638cd0f355aa6fda9dcdeb837 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 22 May 2024 22:07:30 +0530 Subject: [PATCH 11/22] Build package with `nox` in workflow --- .github/workflows/build_package.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 9c91cd1e..2f1d153d 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -1,4 +1,4 @@ -name: Publish +name: Build Package on: pull_request: @@ -25,7 +25,7 @@ jobs: python-version: "3.10" - name: Install build dependencies - run: pip install build + run: pip install nox - name: Set BUILD_RAGNA_BASE environment variable run: | @@ -37,8 +37,12 @@ jobs: - name: Build distribution run: | - echo "Building package: ${{ matrix.package }}" - python -m build + if [ "${{ matrix.package }}" == "ragna-base" ]; then + echo "Building package with nox -s build-base" + nox -s build-base + else + echo "Building package with nox -s build" + nox -s build - name: Verify package run: | From 0028f4ec44c5d62dac47be166d00dfb0ac9e2c5a Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Wed, 22 May 2024 22:13:23 +0530 Subject: [PATCH 12/22] Fix syntax error in workflow --- .github/workflows/build_package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 2f1d153d..674de4fa 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -43,6 +43,7 @@ jobs: else echo "Building package with nox -s build" nox -s build + fi - name: Verify package run: | From fab9c7df896fa90dfdd1c5bbd183861f234b93d2 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Thu, 30 May 2024 16:08:00 +0530 Subject: [PATCH 13/22] Delete splitted pyprojects & Switch to traditional build for now --- .github/workflows/build_package.yml | 13 +-- scripts/pyproject-base.toml | 142 ---------------------------- scripts/pyproject.toml | 142 ---------------------------- 3 files changed, 4 insertions(+), 293 deletions(-) delete mode 100644 scripts/pyproject-base.toml delete mode 100644 scripts/pyproject.toml diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 674de4fa..9c91cd1e 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -1,4 +1,4 @@ -name: Build Package +name: Publish on: pull_request: @@ -25,7 +25,7 @@ jobs: python-version: "3.10" - name: Install build dependencies - run: pip install nox + run: pip install build - name: Set BUILD_RAGNA_BASE environment variable run: | @@ -37,13 +37,8 @@ jobs: - name: Build distribution run: | - if [ "${{ matrix.package }}" == "ragna-base" ]; then - echo "Building package with nox -s build-base" - nox -s build-base - else - echo "Building package with nox -s build" - nox -s build - fi + echo "Building package: ${{ matrix.package }}" + python -m build - name: Verify package run: | diff --git a/scripts/pyproject-base.toml b/scripts/pyproject-base.toml deleted file mode 100644 index 8cc5f9c3..00000000 --- a/scripts/pyproject-base.toml +++ /dev/null @@ -1,142 +0,0 @@ -[build-system] -requires = [ - "setuptools>=45", - "setuptools_scm[toml]>=6.2", -] -build-backend = "setuptools.build_meta" - -[project] -name = "ragna-base" -description = "RAG orchestration framework" -license = {file = "LICENSE"} -authors = [ - { name = "Ragna Development Team", email = "connect@quansight.com" }, -] -readme = "README.md" -classifiers = [ - "Development Status :: 4 - Beta", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", -] -requires-python = ">=3.9" -dynamic = ["dependencies", "optional-dependencies", "version"] - -[project.urls] -Homepage = "https://ragna.chat" -Documentation = "https://ragna.chat" -Changelog = "https://ragna.chat/en/stable/references/release-notes/" -Repository = "https://github.com/Quansight/ragna" - -[tool.setuptools.dynamic] -dependencies = {file = ["requirements-base.txt"]} - -[tool.setuptools_scm] -write_to = "ragna/_version.py" -version_scheme = "release-branch-semver" -local_scheme = "node-and-timestamp" - -[project.scripts] -ragna = "ragna.__main__:app" - -[tool.setuptools.packages.find] -include = [ - "ragna*", -] - -[tool.ruff.lint] -select = [ - "E", - "F", - # import sorting - "I001" -] -# Ignore line too long, because due to black, the error can only occur for strings -ignore = ["E501"] - -[tool.ruff.lint.per-file-ignores] -# ignore unused imports and imports not at the top of the file in __init__.py files -"__init__.py" = ["F401", "E402"] -# The examples often have imports below the top of the file to follow the narrative -"docs/examples/**/*.py" = ["E402", "F704", "I001"] -"docs/tutorials/**/*.py" = ["E402", "F704", "I001"] - -[tool.pytest.ini_options] -minversion = "6.0" -addopts = "-ra --tb=short --asyncio-mode=auto" -testpaths = [ - "tests", -] -filterwarnings = [ - "error", - "ignore::ResourceWarning", - # https://github.com/lancedb/lancedb/issues/1296 - "ignore:Function 'semver.parse_version_info' is deprecated:DeprecationWarning" -] -xfail_strict = true - -[tool.mypy] - -files = "ragna" - -plugins = [ - "sqlmypy", -] - -show_error_codes = true -pretty = true - -disallow_untyped_calls = true -disallow_untyped_defs = true -disallow_incomplete_defs = true - -allow_redefinition = true -no_implicit_optional = true - -warn_redundant_casts = true -warn_unused_ignores = true - -warn_return_any = true -warn_unused_configs = true - -[[tool.mypy.overrides]] -module = [ - "ragna.deploy._ui.*", -] -disallow_untyped_calls = false -disallow_untyped_defs = false -disallow_incomplete_defs = false - -[[tool.mypy.overrides]] -module = [ - "docx", - "fitz", - "ijson", - "lancedb", - "param", - "pptx", - "pyarrow", - "sentence_transformers", -] -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = [ - "ragna.deploy._api.orm", -] -# Our ORM schema doesn't really work with mypy. There are some other ways to define it -# to play ball. We should do that in the future. -disable_error_code = [ - "var-annotated", -] - -[[tool.mypy.overrides]] -# It is a fundamental feature of the components to request more parameters than the base -# class. Thus, we just silence mypy here. -module = [ - "ragna.source_storages.*", - "ragna.assistants.*" -] -disable_error_code = [ - "override", -] diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml deleted file mode 100644 index a6f1fbb2..00000000 --- a/scripts/pyproject.toml +++ /dev/null @@ -1,142 +0,0 @@ -[build-system] -requires = [ - "setuptools>=45", - "setuptools_scm[toml]>=6.2", -] -build-backend = "setuptools.build_meta" - -[project] -name = "Ragna" -description = "RAG orchestration framework" -license = {file = "LICENSE"} -authors = [ - { name = "Ragna Development Team", email = "connect@quansight.com" }, -] -readme = "README.md" -classifiers = [ - "Development Status :: 4 - Beta", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", -] -requires-python = ">=3.9" -dynamic = ["dependencies", "optional-dependencies", "version"] - -[project.urls] -Homepage = "https://ragna.chat" -Documentation = "https://ragna.chat" -Changelog = "https://ragna.chat/en/stable/references/release-notes/" -Repository = "https://github.com/Quansight/ragna" - -[tool.setuptools.dynamic] -dependencies = {file = ["requirements.txt"]} - -[tool.setuptools_scm] -write_to = "ragna/_version.py" -version_scheme = "release-branch-semver" -local_scheme = "node-and-timestamp" - -[project.scripts] -ragna = "ragna.__main__:app" - -[tool.setuptools.packages.find] -include = [ - "ragna*", -] - -[tool.ruff.lint] -select = [ - "E", - "F", - # import sorting - "I001" -] -# Ignore line too long, because due to black, the error can only occur for strings -ignore = ["E501"] - -[tool.ruff.lint.per-file-ignores] -# ignore unused imports and imports not at the top of the file in __init__.py files -"__init__.py" = ["F401", "E402"] -# The examples often have imports below the top of the file to follow the narrative -"docs/examples/**/*.py" = ["E402", "F704", "I001"] -"docs/tutorials/**/*.py" = ["E402", "F704", "I001"] - -[tool.pytest.ini_options] -minversion = "6.0" -addopts = "-ra --tb=short --asyncio-mode=auto" -testpaths = [ - "tests", -] -filterwarnings = [ - "error", - "ignore::ResourceWarning", - # https://github.com/lancedb/lancedb/issues/1296 - "ignore:Function 'semver.parse_version_info' is deprecated:DeprecationWarning" -] -xfail_strict = true - -[tool.mypy] - -files = "ragna" - -plugins = [ - "sqlmypy", -] - -show_error_codes = true -pretty = true - -disallow_untyped_calls = true -disallow_untyped_defs = true -disallow_incomplete_defs = true - -allow_redefinition = true -no_implicit_optional = true - -warn_redundant_casts = true -warn_unused_ignores = true - -warn_return_any = true -warn_unused_configs = true - -[[tool.mypy.overrides]] -module = [ - "ragna.deploy._ui.*", -] -disallow_untyped_calls = false -disallow_untyped_defs = false -disallow_incomplete_defs = false - -[[tool.mypy.overrides]] -module = [ - "docx", - "fitz", - "ijson", - "lancedb", - "param", - "pptx", - "pyarrow", - "sentence_transformers", -] -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = [ - "ragna.deploy._api.orm", -] -# Our ORM schema doesn't really work with mypy. There are some other ways to define it -# to play ball. We should do that in the future. -disable_error_code = [ - "var-annotated", -] - -[[tool.mypy.overrides]] -# It is a fundamental feature of the components to request more parameters than the base -# class. Thus, we just silence mypy here. -module = [ - "ragna.source_storages.*", - "ragna.assistants.*" -] -disable_error_code = [ - "override", -] From c396b55a5349a67bd99fb788a254220dab872528 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Thu, 30 May 2024 16:37:03 +0530 Subject: [PATCH 14/22] Try using pipx to build package --- .github/workflows/build_package.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 9c91cd1e..999b2bca 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -1,4 +1,4 @@ -name: Publish +name: Build Package on: pull_request: @@ -24,8 +24,11 @@ jobs: with: python-version: "3.10" - - name: Install build dependencies - run: pip install build + - name: Install pipx + run: python -m pip install --user pipx + + - name: Install build using pipx + run: pipx install build - name: Set BUILD_RAGNA_BASE environment variable run: | @@ -38,7 +41,7 @@ jobs: - name: Build distribution run: | echo "Building package: ${{ matrix.package }}" - python -m build + pipx run build - name: Verify package run: | From bd654873bca7ca48349bdd23d356de10067648d9 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Fri, 31 May 2024 02:29:08 +0530 Subject: [PATCH 15/22] Get rid of `noxfile` & `setup` try using helper script instead --- .github/workflows/build_package.yml | 21 ++++++++------------ noxfile.py | 27 -------------------------- scripts/build_helper.py | 30 +++++++++++++++++++++++++++++ setup.py | 28 --------------------------- 4 files changed, 38 insertions(+), 68 deletions(-) delete mode 100644 noxfile.py create mode 100644 scripts/build_helper.py delete mode 100644 setup.py diff --git a/.github/workflows/build_package.yml b/.github/workflows/build_package.yml index 999b2bca..4134fb3f 100644 --- a/.github/workflows/build_package.yml +++ b/.github/workflows/build_package.yml @@ -24,24 +24,19 @@ jobs: with: python-version: "3.10" - - name: Install pipx - run: python -m pip install --user pipx - - - name: Install build using pipx - run: pipx install build - - - name: Set BUILD_RAGNA_BASE environment variable + - name: Install build dependencies run: | - if [ "${{ matrix.package }}" == "ragna-base" ]; then - echo "BUILD_RAGNA_BASE=1" >> $GITHUB_ENV - else - echo "BUILD_RAGNA_BASE=" >> $GITHUB_ENV - fi + python -m pip install --upgrade pip + python -m pip install build toml + + - name: Modify pyproject.toml + run: python scripts/build_helper.py ${{ matrix.package }} - name: Build distribution run: | echo "Building package: ${{ matrix.package }}" - pipx run build + python -m build + echo "Build complete for ${{ matrix.package }}" - name: Verify package run: | diff --git a/noxfile.py b/noxfile.py deleted file mode 100644 index b6ee384a..00000000 --- a/noxfile.py +++ /dev/null @@ -1,27 +0,0 @@ -import os - -import nox - - -@nox.session(name="build") -def build(session): - setup_script = "scripts/pyproject.toml" - symlink_pyproject(setup_script) - session.install("build") - session.run("python", "-m", "build") - - -@nox.session(name="build-base") -def build_bar(session): - setup_script = "scripts/pyproject-base.toml" - symlink_pyproject(setup_script) - session.install("build") - session.run("python", "-m", "build") - - -def symlink_pyproject(script): - if os.path.exists(script): - # Remove existing pyproject.toml if it exists - if os.path.exists("pyproject.toml"): - os.remove("pyproject.toml") - os.symlink(script, "pyproject.toml") diff --git a/scripts/build_helper.py b/scripts/build_helper.py new file mode 100644 index 00000000..87ff3386 --- /dev/null +++ b/scripts/build_helper.py @@ -0,0 +1,30 @@ +import toml + + +def modify_pyproject(package_name): + with open("pyproject.toml", "r") as f: + pyproject_data = toml.load(f) + + pyproject_data["project"]["name"] = package_name + + if package_name == "ragna-base": + pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = { + "file": ["requirements-base.txt"] + } + else: + pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = { + "file": ["requirements.txt"] + } + + with open("pyproject.toml", "w") as f: + toml.dump(pyproject_data, f) + + +if __name__ == "__main__": + import sys + + package_name = sys.argv[1] + if package_name not in ["ragna", "ragna-base"]: + print("Invalid package name. Must be 'ragna' or 'ragna-base'.") + sys.exit(1) + modify_pyproject(package_name) diff --git a/setup.py b/setup.py deleted file mode 100644 index 6861a740..00000000 --- a/setup.py +++ /dev/null @@ -1,28 +0,0 @@ -import os - -from setuptools import setup - -print("BUILD_RAGNA_BASE=", os.environ["BUILD_RAGNA_BASE"]) - -with open("requirements.txt") as f: - ragna_dependencies = f.read().splitlines() - -with open("requirements-base.txt") as f: - base_dependencies = f.read().splitlines() - -is_base_build = os.environ.get("BUILD_RAGNA_BASE") - -name = "ragna-base" if os.environ.get("BUILD_RAGNA_BASE") else "ragna" -dependencies = ( - base_dependencies if os.environ.get("BUILD_RAGNA_BASE") else ragna_dependencies -) - -print(f"Building package: {name}") -print( - f"Using dependencies from {'requirements-base.txt' if is_base_build else 'requirements.txt'}" -) - -setup( - name=name, - install_requires=dependencies, -) From 241fab57470bbe9799fd8b0806dcbeb9bd5e0883 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Sun, 2 Jun 2024 18:29:33 +0530 Subject: [PATCH 16/22] Fix dependency conflicts for docker build Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- pyproject.toml | 2 +- requirements-base.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a6f1fbb2..38ab99f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] requires-python = ">=3.9" -dynamic = ["dependencies", "optional-dependencies", "version"] +dynamic = ["version"] [project.urls] Homepage = "https://ragna.chat" diff --git a/requirements-base.txt b/requirements-base.txt index 13eb3e66..bdc51b3f 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -4,7 +4,7 @@ fastapi httpx importlib_metadata>=4.6; python_version<'3.10' packaging -panel==1.3.8 +panel==1.4.2 pydantic>=2 pydantic-core pydantic-settings>=2 diff --git a/requirements.txt b/requirements.txt index 7da3dd6f..2c110afa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ fastapi httpx importlib_metadata>=4.6; python_version<'3.10' packaging -panel==1.3.8 +panel==1.4.2 pydantic>=2 pydantic-core pydantic-settings>=2 From 855025d4ba4f7aad02063a7e3d4a3cde97b9558e Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Sun, 2 Jun 2024 19:35:42 +0530 Subject: [PATCH 17/22] Try fixing docker lockfile dailure --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 38ab99f9..37cf8913 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] requires-python = ">=3.9" -dynamic = ["version"] +dynamic = ["dependencies", "version"] [project.urls] Homepage = "https://ragna.chat" From f042769dc620e38ca2f1dba0ef781ba193f51840 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Sun, 2 Jun 2024 20:23:14 +0530 Subject: [PATCH 18/22] Modity `update_optional_dependencies` to update `requirements.txt` --- scripts/update_optional_dependencies.py | 35 +++++++++++++++---------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/scripts/update_optional_dependencies.py b/scripts/update_optional_dependencies.py index a99f206c..9026e7cd 100644 --- a/scripts/update_optional_dependencies.py +++ b/scripts/update_optional_dependencies.py @@ -2,15 +2,13 @@ from functools import reduce from pathlib import Path -import tomlkit -import tomlkit.items from packaging.requirements import Requirement import ragna from ragna.core import Assistant, SourceStorage HERE = Path(__file__).parent -PYPROJECT_TOML = HERE / ".." / "pyproject.toml" +REQUIREMENTS_TXT = HERE / ".." / "requirements.txt" def main(): @@ -18,7 +16,7 @@ def main(): extract_builtin_document_handler_requirements(), extract_builtin_component_requirements(), ) - update_pyproject_toml(optional_dependencies) + update_requirements_txt(optional_dependencies) def make_optional_dependencies(*optional_requirements): @@ -57,18 +55,27 @@ def extract_builtin_component_requirements(): return dict(requirements) -def update_pyproject_toml(optional_dependencies): - with open(PYPROJECT_TOML) as file: - document = tomlkit.load(file) +def update_requirements_txt(optional_dependencies): + existing_dependencies = set() - document["project"]["optional-dependencies"]["all"] = tomlkit.items.Array( - list(map(tomlkit.items.String.from_raw, optional_dependencies)), - trivia=tomlkit.items.Trivia(), - multiline=True, - ) + # Read existing dependencies from requirements.txt + if REQUIREMENTS_TXT.exists(): + with open(REQUIREMENTS_TXT, "r") as file: + existing_dependencies = set(line.strip() for line in file) + + new_dependencies = [] + + # Check if each optional dependency already exists + for dependency in optional_dependencies: + if dependency not in existing_dependencies: + new_dependencies.append(dependency) + existing_dependencies.add(dependency) - with open(PYPROJECT_TOML, "w") as file: - tomlkit.dump(document, file) + # Append new dependencies to requirements.txt + if new_dependencies: + with open(REQUIREMENTS_TXT, "a") as file: + file.write("\n".join(new_dependencies)) + file.write("\n") def append_version_specifiers(version_specifiers, obj): From a7c9a01447f796837d974ccf3be41ee60c3e53ed Mon Sep 17 00:00:00 2001 From: Arjun Verma Date: Thu, 6 Jun 2024 16:00:44 +0530 Subject: [PATCH 19/22] Apply suggestions from code review Co-authored-by: Philip Meier --- scripts/build_helper.py | 9 ++------- scripts/update_optional_dependencies.py | 3 ++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/build_helper.py b/scripts/build_helper.py index 87ff3386..7d525e06 100644 --- a/scripts/build_helper.py +++ b/scripts/build_helper.py @@ -7,13 +7,8 @@ def modify_pyproject(package_name): pyproject_data["project"]["name"] = package_name - if package_name == "ragna-base": - pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = { - "file": ["requirements-base.txt"] - } - else: - pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = { - "file": ["requirements.txt"] + pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = { + "file": ["requirements-base.txt" if package_name == "ragna-base" else "requirements.txt"] } with open("pyproject.toml", "w") as f: diff --git a/scripts/update_optional_dependencies.py b/scripts/update_optional_dependencies.py index 9026e7cd..5d5d8eb2 100644 --- a/scripts/update_optional_dependencies.py +++ b/scripts/update_optional_dependencies.py @@ -8,7 +8,8 @@ from ragna.core import Assistant, SourceStorage HERE = Path(__file__).parent -REQUIREMENTS_TXT = HERE / ".." / "requirements.txt" +PROJECT_ROOT = HERE.parent +REQUIREMENTS_TXT = PROJECT_ROOT / "requirements.txt" def main(): From 2e7022900c7104edd2ec0db052abfd0807038d08 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Thu, 6 Jun 2024 20:59:53 +0530 Subject: [PATCH 20/22] Modify `update_requirements_txt` -> `create_requirements_txt` Incorporate suggestions from Code Review --- requirements.txt | 24 +++++++++---------- scripts/build_helper.py | 23 ++++++++++++------ scripts/update_optional_dependencies.py | 31 +++++++++++-------------- 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2c110afa..7a0931d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,29 @@ +PyJWT aiofiles +chromadb>=0.4.13 emoji fastapi httpx +httpx_sse +ijson importlib_metadata>=4.6; python_version<'3.10' +lancedb>=0.2 packaging panel==1.4.2 -pydantic>=2 +pyarrow pydantic-core pydantic-settings>=2 -PyJWT +pydantic>=2 +pymupdf>=1.23.6 +python-docx python-multipart -redis +python-pptx questionary +redis rich sqlalchemy>=2 starlette +tiktoken tomlkit typer uvicorn -chromadb>=0.4.13 -httpx_sse -ijson -lancedb>=0.2 -pyarrow -pymupdf>=1.23.6 -python-docx -python-pptx -tiktoken diff --git a/scripts/build_helper.py b/scripts/build_helper.py index 7d525e06..cad214d9 100644 --- a/scripts/build_helper.py +++ b/scripts/build_helper.py @@ -1,23 +1,32 @@ +import sys +from pathlib import Path + import toml +HERE = Path(__file__).parent +PROJECT_ROOT = HERE.parent +PYPROJECT_TOML = PROJECT_ROOT / "pyproject.toml" + def modify_pyproject(package_name): - with open("pyproject.toml", "r") as f: + with open(PYPROJECT_TOML, "r") as f: pyproject_data = toml.load(f) pyproject_data["project"]["name"] = package_name pyproject_data["tool"]["setuptools"]["dynamic"]["dependencies"] = { - "file": ["requirements-base.txt" if package_name == "ragna-base" else "requirements.txt"] - } - - with open("pyproject.toml", "w") as f: + "file": [ + "requirements-base.txt" + if package_name == "ragna-base" + else "requirements.txt" + ] + } + + with open(PYPROJECT_TOML, "w") as f: toml.dump(pyproject_data, f) if __name__ == "__main__": - import sys - package_name = sys.argv[1] if package_name not in ["ragna", "ragna-base"]: print("Invalid package name. Must be 'ragna' or 'ragna-base'.") diff --git a/scripts/update_optional_dependencies.py b/scripts/update_optional_dependencies.py index 5d5d8eb2..768e722e 100644 --- a/scripts/update_optional_dependencies.py +++ b/scripts/update_optional_dependencies.py @@ -10,6 +10,7 @@ HERE = Path(__file__).parent PROJECT_ROOT = HERE.parent REQUIREMENTS_TXT = PROJECT_ROOT / "requirements.txt" +REQUIREMENTS_BASE_TXT = PROJECT_ROOT / "requirements-base.txt" def main(): @@ -17,7 +18,7 @@ def main(): extract_builtin_document_handler_requirements(), extract_builtin_component_requirements(), ) - update_requirements_txt(optional_dependencies) + create_requirements_txt(optional_dependencies) def make_optional_dependencies(*optional_requirements): @@ -56,27 +57,23 @@ def extract_builtin_component_requirements(): return dict(requirements) -def update_requirements_txt(optional_dependencies): +def create_requirements_txt(optional_dependencies): + # Read existing dependencies from requirements-base.txt existing_dependencies = set() - - # Read existing dependencies from requirements.txt - if REQUIREMENTS_TXT.exists(): - with open(REQUIREMENTS_TXT, "r") as file: + if REQUIREMENTS_BASE_TXT.exists(): + with open(REQUIREMENTS_BASE_TXT, "r") as file: existing_dependencies = set(line.strip() for line in file) - new_dependencies = [] + # Add optional dependencies + all_dependencies = existing_dependencies.union(optional_dependencies) - # Check if each optional dependency already exists - for dependency in optional_dependencies: - if dependency not in existing_dependencies: - new_dependencies.append(dependency) - existing_dependencies.add(dependency) + # Sort the dependencies to maintain a consistent order + sorted_dependencies = sorted(all_dependencies) - # Append new dependencies to requirements.txt - if new_dependencies: - with open(REQUIREMENTS_TXT, "a") as file: - file.write("\n".join(new_dependencies)) - file.write("\n") + # Overwrite the requirements.txt file with the sorted dependencies + with open(REQUIREMENTS_TXT, "w") as file: + file.write("\n".join(sorted_dependencies)) + file.write("\n") def append_version_specifiers(version_specifiers, obj): From 9b0f1c27f8f175e631acd771599be1a106ddf399 Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Thu, 6 Jun 2024 21:04:33 +0530 Subject: [PATCH 21/22] Bump `setuptools` & `setuptools_scm` --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 37cf8913..9284cd3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] requires = [ - "setuptools>=45", - "setuptools_scm[toml]>=6.2", + "setuptools>=66", + "setuptools_scm[toml]>=8", ] build-backend = "setuptools.build_meta" From ed1cfdeaa1051e1c64be26257a5a0d3032a8590b Mon Sep 17 00:00:00 2001 From: "arjxn.py" Date: Sun, 30 Jun 2024 04:33:34 +0530 Subject: [PATCH 22/22] Replace base dependencies in `requirements.txt` by `ragna-base` --- requirements.txt | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7a0931d1..b225b278 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,10 @@ -PyJWT -aiofiles +ragna-base chromadb>=0.4.13 -emoji -fastapi -httpx httpx_sse ijson -importlib_metadata>=4.6; python_version<'3.10' lancedb>=0.2 -packaging -panel==1.4.2 pyarrow -pydantic-core -pydantic-settings>=2 -pydantic>=2 pymupdf>=1.23.6 python-docx -python-multipart python-pptx -questionary -redis -rich -sqlalchemy>=2 -starlette tiktoken -tomlkit -typer -uvicorn