diff --git a/.release-please-manifest.json b/.release-please-manifest.json index c373724d..46b9b6b2 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.1.0-alpha.8" + ".": "0.1.0-alpha.9" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index c2bf50d2..9947ea6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.1.0-alpha.9 (2025-05-31) + +Full Changelog: [v0.1.0-alpha.8...v0.1.0-alpha.9](https://github.com/togethercomputer/together-py/compare/v0.1.0-alpha.8...v0.1.0-alpha.9) + +### Features + +* **api:** get file upload working ([cb8b8b8](https://github.com/togethercomputer/together-py/commit/cb8b8b86974721c2b2366e8481b88b3cb4851f0c)) +* **api:** move upload to be a method of existing files resource ([b7c43be](https://github.com/togethercomputer/together-py/commit/b7c43be446e48390528994ee5a070699c490cec4)) + + +### Bug Fixes + +* **api:** correct file reroute handling, error message ([b8bc101](https://github.com/togethercomputer/together-py/commit/b8bc1010e047ba0b1bd75a311cb1220f13366f04)) + ## 0.1.0-alpha.8 (2025-05-29) Full Changelog: [v0.1.0-alpha.7...v0.1.0-alpha.8](https://github.com/togethercomputer/together-py/compare/v0.1.0-alpha.7...v0.1.0-alpha.8) diff --git a/pyproject.toml b/pyproject.toml index 4082c986..b18ae892 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "together" -version = "0.1.0-alpha.8" +version = "0.1.0-alpha.9" description = "The official Python library for the together API" dynamic = ["readme"] license = "Apache-2.0" diff --git a/src/together/_base_client.py b/src/together/_base_client.py index ee2f5115..ad80ed98 100644 --- a/src/together/_base_client.py +++ b/src/together/_base_client.py @@ -960,6 +960,9 @@ def request( if self.custom_auth is not None: kwargs["auth"] = self.custom_auth + if options.follow_redirects is not None: + kwargs["follow_redirects"] = options.follow_redirects + log.debug("Sending HTTP Request: %s %s", request.method, request.url) response = None @@ -1460,6 +1463,9 @@ async def request( if self.custom_auth is not None: kwargs["auth"] = self.custom_auth + if options.follow_redirects is not None: + kwargs["follow_redirects"] = options.follow_redirects + log.debug("Sending HTTP Request: %s %s", request.method, request.url) response = None diff --git a/src/together/_models.py b/src/together/_models.py index 798956f1..4f214980 100644 --- a/src/together/_models.py +++ b/src/together/_models.py @@ -737,6 +737,7 @@ class FinalRequestOptionsInput(TypedDict, total=False): idempotency_key: str json_data: Body extra_json: AnyMapping + follow_redirects: bool @final @@ -750,6 +751,7 @@ class FinalRequestOptions(pydantic.BaseModel): files: Union[HttpxRequestFiles, None] = None idempotency_key: Union[str, None] = None post_parser: Union[Callable[[Any], Any], NotGiven] = NotGiven() + follow_redirects: Union[bool, None] = None # It should be noted that we cannot use `json` here as that would override # a BaseModel method in an incompatible fashion. diff --git a/src/together/_types.py b/src/together/_types.py index 95b4969c..58fd7c58 100644 --- a/src/together/_types.py +++ b/src/together/_types.py @@ -100,6 +100,7 @@ class RequestOptions(TypedDict, total=False): params: Query extra_json: AnyMapping idempotency_key: str + follow_redirects: bool # Sentinel class used until PEP 0661 is accepted @@ -215,3 +216,4 @@ class _GenericAlias(Protocol): class HttpxSendArgs(TypedDict, total=False): auth: httpx.Auth + follow_redirects: bool diff --git a/src/together/_version.py b/src/together/_version.py index 9dbfbc54..b76f4a44 100644 --- a/src/together/_version.py +++ b/src/together/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "together" -__version__ = "0.1.0-alpha.8" # x-release-please-version +__version__ = "0.1.0-alpha.9" # x-release-please-version diff --git a/src/together/lib/cli/api/files.py b/src/together/lib/cli/api/files.py index 3b56aa1d..4cc9ed20 100644 --- a/src/together/lib/cli/api/files.py +++ b/src/together/lib/cli/api/files.py @@ -29,8 +29,8 @@ def files(_ctx: click.Context) -> None: @click.option( "--purpose", type=str, - default="fine-tunes", - help="Purpose of file upload. Acceptable values in enum `together.types.FilePurpose`. Defaults to `fine-tunes`.", + default="fine-tune", + help="Purpose of file upload. Acceptable values in enum `together.types.FilePurpose`. Defaults to `fine-tune`.", ) @click.option( "--check/--no-check", @@ -42,7 +42,7 @@ def upload(ctx: click.Context, file: pathlib.Path, purpose: str, check: bool) -> client: Together = ctx.obj - response = client.files.upload(file=file, purpose=purpose, check=check) + response = client.files.upload_file(file=file, purpose=purpose, check=check) click.echo(json.dumps(response.model_dump(exclude_none=True), indent=4)) diff --git a/src/together/lib/resources/files.py b/src/together/lib/resources/files.py index bb351f0b..bbca6887 100644 --- a/src/together/lib/resources/files.py +++ b/src/together/lib/resources/files.py @@ -4,8 +4,9 @@ import stat import uuid import shutil +import logging import tempfile -from typing import Tuple +from typing import IO, Tuple, cast from pathlib import Path from functools import partial @@ -21,6 +22,8 @@ from ..types.error import DownloadError, FileTypeError from ..._exceptions import APIStatusError, AuthenticationError +log: logging.Logger = logging.getLogger(__name__) + def chmod_and_replace(src: Path, dst: Path) -> None: """Set correct permission before moving a blob from tmp directory to cache dir. @@ -225,6 +228,7 @@ def get_upload_url( path=url, cast_to=httpx.Response, body=data, + options={"headers": {"Content-Type": "multipart/form-data"}, "follow_redirects": False}, ) except APIStatusError as e: if e.response.status_code == 401: @@ -235,15 +239,14 @@ def get_upload_url( response=e.response, body=e.body, ) from e - raise - # Raise error for non 302 status codes - if response.status_code != 302: - raise APIStatusError( - f"Unexpected error raised by endpoint: {response.content.decode()}, headers: {response.headers}", - response=response, - body=response.content.decode(), - ) + if e.response.status_code != 302: + raise APIStatusError( + f"Unexpected error raised by endpoint: {e.response.content.decode()}, headers: {e.response.headers}", + response=e.response, + body=e.response.content.decode(), + ) from e + response = e.response redirect_url = response.headers["Location"] file_id = response.headers["X-Together-File-Id"] @@ -263,21 +266,19 @@ def upload( url: str, file: Path, purpose: FilePurpose, - redirect: bool = False, ) -> FileRetrieveResponse: file_id = None redirect_url = None - if redirect: - if file.suffix == ".jsonl": - filetype = "jsonl" - elif file.suffix == ".parquet": - filetype = "parquet" - else: - raise FileTypeError( - f"Unknown extension of file {file}. Only files with extensions .jsonl and .parquet are supported." - ) - redirect_url, file_id = self.get_upload_url(url, file, purpose, filetype) # type: ignore + if file.suffix == ".jsonl": + filetype = "jsonl" + elif file.suffix == ".parquet": + filetype = "parquet" + else: + raise FileTypeError( + f"Unknown extension of file {file}. Only files with extensions .jsonl and .parquet are supported." + ) + redirect_url, file_id = self.get_upload_url(url, file, purpose, filetype) # type: ignore file_size = os.stat(file.as_posix()).st_size @@ -289,33 +290,32 @@ def upload( disable=bool(DISABLE_TQDM), ) as pbar: with file.open("rb") as f: - wrapped_file = CallbackIOWrapper(pbar.update, f, "read") + wrapped_file = cast(IO[bytes], CallbackIOWrapper(pbar.update, f, "read")) - if redirect: - assert redirect_url is not None - callback_response = self._client.put( - cast_to=httpx.Response, - path=redirect_url, - body=wrapped_file, - ) - else: - response = self._client.put( - cast_to=FileRetrieveResponse, - path=url, - body=wrapped_file, - ) + assert redirect_url is not None + callback_response = self._client._client.put( + url=redirect_url, + content=wrapped_file.read(), + ) + log.debug( + 'HTTP Response: %s %s "%i %s" %s', + "put", + redirect_url, + callback_response.status_code, + callback_response.reason_phrase, + callback_response.headers, + ) - if redirect: - assert isinstance(callback_response, httpx.Response) # type: ignore + assert isinstance(callback_response, httpx.Response) # type: ignore - if not callback_response.status_code == 200: - raise APIStatusError( - f"Error during file upload: {callback_response.content.decode()}, headers: {callback_response.headers}", - response=callback_response, - body=callback_response.content.decode(), - ) + if not callback_response.status_code == 200: + raise APIStatusError( + f"Error during file upload: {callback_response.content.decode()}, headers: {callback_response.headers}", + response=callback_response, + body=callback_response.content.decode(), + ) - response = self.callback(f"{url}/{file_id}/preprocess") + response = self.callback(f"{url}/{file_id}/preprocess") assert isinstance(response, FileRetrieveResponse) # type: ignore @@ -379,21 +379,19 @@ async def upload( url: str, file: Path, purpose: FilePurpose, - redirect: bool = False, ) -> FileRetrieveResponse: file_id = None redirect_url = None - if redirect: - if file.suffix == ".jsonl": - filetype = "jsonl" - elif file.suffix == ".parquet": - filetype = "parquet" - else: - raise FileTypeError( - f"Unknown extension of file {file}. Only files with extensions .jsonl and .parquet are supported." - ) - redirect_url, file_id = await self.get_upload_url(url, file, purpose, filetype) # type: ignore + if file.suffix == ".jsonl": + filetype = "jsonl" + elif file.suffix == ".parquet": + filetype = "parquet" + else: + raise FileTypeError( + f"Unknown extension of file {file}. Only files with extensions .jsonl and .parquet are supported." + ) + redirect_url, file_id = await self.get_upload_url(url, file, purpose, filetype) # type: ignore file_size = os.stat(file.as_posix()).st_size @@ -405,33 +403,32 @@ async def upload( disable=bool(DISABLE_TQDM), ) as pbar: with file.open("rb") as f: - wrapped_file = CallbackIOWrapper(pbar.update, f, "read") + wrapped_file = cast(IO[bytes], CallbackIOWrapper(pbar.update, f, "read")) - if redirect: - assert redirect_url is not None - callback_response = self._client.put( - cast_to=httpx.Response, - path=redirect_url, - body=wrapped_file, - ) - else: - response = self._client.put( - cast_to=FileRetrieveResponse, - path=url, - body=wrapped_file, - ) + assert redirect_url is not None + callback_response = await self._client._client.put( + url=redirect_url, + content=wrapped_file.read(), + ) + log.debug( + 'HTTP Response: %s %s "%i %s" %s', + "put", + redirect_url, + callback_response.status_code, + callback_response.reason_phrase, + callback_response.headers, + ) - if redirect: - assert isinstance(callback_response, httpx.Response) # type: ignore + assert isinstance(callback_response, httpx.Response) # type: ignore - if not callback_response.status_code == 200: - raise APIStatusError( - f"Error during file upload: {callback_response.content.decode()}, headers: {callback_response.headers}", - response=callback_response, - body=callback_response.content.decode(), - ) + if not callback_response.status_code == 200: + raise APIStatusError( + f"Error during file upload: {callback_response.content.decode()}, headers: {callback_response.headers}", + response=callback_response, + body=callback_response.content.decode(), + ) - response = self.callback(f"{url}/{file_id}/preprocess") + response = self.callback(f"{url}/{file_id}/preprocess") assert isinstance(response, FileRetrieveResponse) # type: ignore diff --git a/src/together/lib/types/error.py b/src/together/lib/types/error.py index c482216f..36d6dfbb 100644 --- a/src/together/lib/types/error.py +++ b/src/together/lib/types/error.py @@ -1,23 +1,9 @@ -from typing import Any - from ..._exceptions import TogetherError class DownloadError(TogetherError): - def __init__( - self, - message: str, - **kwargs: Any, - ) -> None: - self.message = message - super().__init__(**kwargs) + pass class FileTypeError(TogetherError): - def __init__( - self, - message: str, - **kwargs: Any, - ) -> None: - self.message = message - super().__init__(**kwargs) + pass diff --git a/src/together/resources/files.py b/src/together/resources/files.py index 4fb0c3d1..04ad170b 100644 --- a/src/together/resources/files.py +++ b/src/together/resources/files.py @@ -140,7 +140,7 @@ def delete( cast_to=FileDeleteResponse, ) - def upload( + def upload_file( self, file: Path | str, *, @@ -162,7 +162,7 @@ def upload( purpose = cast(FilePurpose, purpose) - return upload_manager.upload("files", file, purpose=purpose, redirect=True) + return upload_manager.upload("files", file, purpose=purpose) def content( self, @@ -304,7 +304,7 @@ async def delete( cast_to=FileDeleteResponse, ) - async def upload( + async def upload_file( self, file: Path | str, *, @@ -326,7 +326,7 @@ async def upload( purpose = cast(FilePurpose, purpose) - return await upload_manager.upload("files", file, purpose=purpose, redirect=True) + return await upload_manager.upload("files/upload", file, purpose=purpose) async def content( self, diff --git a/tests/integration/resources/test_files.py b/tests/integration/resources/test_files.py new file mode 100644 index 00000000..6d615d36 --- /dev/null +++ b/tests/integration/resources/test_files.py @@ -0,0 +1,48 @@ +import os +import json +from pathlib import Path + +import pytest + +from together import Together +from together.types import ( + FileRetrieveResponse, +) + + +class TestTogetherFiles: + @pytest.fixture + def sync_together_client(self) -> Together: + """ + Initialize object with mocked API key + """ + TOGETHER_API_KEY = os.getenv("TOGETHER_API_KEY") + return Together(api_key=TOGETHER_API_KEY) + + def test_file_upload_file( + self, + sync_together_client: Together, + tmp_path: Path, + ): + # Mock the post method directly on the client + files = sync_together_client.files + + # Make a temporary file object + file = tmp_path / "valid.jsonl" + content = [{"text": "Hello, world!"}, {"text": "How are you?"}] + with file.open("w") as f: + f.write("\n".join(json.dumps(item) for item in content)) + + # Test run method + response = files.upload_file( + file, + ) + + # Verify the response + assert isinstance(response, FileRetrieveResponse) + assert response.filename == "valid.jsonl" + assert response.file_type == "jsonl" + assert response.line_count == 0 + assert response.object == "file" + assert response.processed == True + assert response.purpose == "fine-tune" diff --git a/tests/unit/test_files_resource.py b/tests/unit/test_files_resource.py new file mode 100644 index 00000000..3efe01d1 --- /dev/null +++ b/tests/unit/test_files_resource.py @@ -0,0 +1,85 @@ +import json +from pathlib import Path + +from httpx import ( + Response, +) +from pytest_mock import MockerFixture + +from together import Together +from together.types import ( + FileRetrieveResponse, +) + + +def test_file_upload_file(mocker: MockerFixture, tmp_path: Path): + # Mock the API requestor + + content = [{"text": "Hello, world!"}, {"text": "How are you?"}] + content_str = "\n".join(json.dumps(item) for item in content) + content_bytes = content_str.encode() + + mock_request = mocker.MagicMock() + mock_request.headers = {} # response.request headers have to be set otherwise it will confuse the framework and not parse the response into an object + + mock_send_response0 = Response( + status_code=302, + headers={ + "Location": "https://3721873h1.r2.cloudflarestorage.com/together-dev//finetune/file-30b2f515-c146-4780-80e6-d8a84f4caaaa", + "X-Together-File-Id": "file-30b2f515-c146-4780-80e6-d8a84f4caaaa", + }, + request=mock_request, + ) + mock_put_response0 = Response( + status_code=200, + request=mock_request, + ) + mock_send_response1 = Response( + status_code=200, + json={ + "id": "file-30b2f515-c146-4780-80e6-d8a84f4caaaa", + "bytes": len(content_str), + "created_at": 1234567890, + "filename": "valid.jsonl", + "FileType": "jsonl", + "LineCount": 0, + "purpose": "fine-tune", + "object": "file", + "Processed": True, + }, + request=mock_request, + ) + + mock_send_requestor = mocker.MagicMock() + mock_send_requestor.side_effect = [mock_send_response0, mock_send_response1] + + mock_put_requestor = mocker.MagicMock() + mock_put_requestor.side_effect = [mock_put_response0] + + # Mock the post method directly on the client + client = Together(api_key="fake_api_key") + mocker.patch.object(client._client, "send", mock_send_requestor) + mocker.patch.object(client._client, "put", mock_put_requestor) + files = client.files + + # Make a temporary file object + file = tmp_path / "valid.jsonl" + with file.open("w") as f: + f.write(content_str) + + # Test run method + response = files.upload_file( + file, + purpose="fine-tune", + ) + + # Verify the response + assert isinstance(response, FileRetrieveResponse) + assert response.filename == "valid.jsonl" + assert response.bytes == len(content_bytes) + assert response.created_at == 1234567890 + assert response.file_type == "jsonl" + assert response.line_count == 0 + assert response.object == "file" + assert response.processed == True + assert response.purpose == "fine-tune"