diff --git a/.github/workflows/ci_docs.yaml b/.github/workflows/ci_docs.yaml index 2c80f6b6e1..d6727fad8c 100644 --- a/.github/workflows/ci_docs.yaml +++ b/.github/workflows/ci_docs.yaml @@ -56,6 +56,5 @@ jobs: if: github.ref == 'refs/heads/main' with: github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: docs/hr/build + publish_dir: docs/build cname: docs.superduperdb.com - enable_jekyll: true diff --git a/Makefile b/Makefile index b23c5bba89..6af49f4c64 100644 --- a/Makefile +++ b/Makefile @@ -62,13 +62,9 @@ install_devkit: ## Add essential development tools ##@ Code Quality gen_docs: ## Generate Docs and API - @echo "===> Generate API docs as Markdown <===" - python docs/hr/content/build_api_docs.py - @echo "Build finished. The HTML pages are in docs/hr/content/build_api_docs" - @echo "===> Generate docusaurus docs and blog-posts <===" - cd docs/hr && npm i --legacy-peer-deps && npm run build - cd ../.. + cd docs && npm i --legacy-peer-deps && npm run build + cd .. @echo "Build finished. The HTML pages are in docs/hr/build" lint-and-type-check: ## Lint and type-check the code diff --git a/docs/content/api/backends/ibis/cursor.md b/docs/content/api/backends/ibis/cursor.md deleted file mode 100644 index 0828f75fee..0000000000 --- a/docs/content/api/backends/ibis/cursor.md +++ /dev/null @@ -1,21 +0,0 @@ -**`superduperdb.backends.ibis.cursor`** - -[Source code](https://github.com/SuperDuperDB/superduperdb/blob/main/superduperdb/backends/ibis/cursor.py) - -## `SuperDuperIbisResult` - -```python -SuperDuperIbisResult(self, - raw_cursor: Any, - id_field: str, - db: Optional[ForwardRef('Datalayer')] = None, - scores: Optional[Dict[str, - float]] = None, - decode_function: Optional[Callable] = None, - _it: int = 0) -> None -``` -SuperDuperIbisResult class for ibis query results. - -SuperDuperIbisResult represents ibis query results with options -to unroll results as i.e pandas. - diff --git a/docs/content/api/base/datalayer.md b/docs/content/api/base/datalayer.md index f62bf12b0f..02c8cc6e56 100644 --- a/docs/content/api/base/datalayer.md +++ b/docs/content/api/base/datalayer.md @@ -9,7 +9,7 @@ Datalayer(self, databackend: superduperdb.backends.base.data_backend.BaseDataBackend, metadata: superduperdb.backends.base.metadata.MetaDataStore, artifact_store: superduperdb.backends.base.artifacts.ArtifactStore, - compute: superduperdb.backends.base.compute.ComputeBackend = ) + compute: superduperdb.backends.base.compute.ComputeBackend = ) ``` | Parameter | Description | |-----------|-------------| diff --git a/docs/content/api/base/enums.md b/docs/content/api/base/enums.md deleted file mode 100644 index fb28c0bbbf..0000000000 --- a/docs/content/api/base/enums.md +++ /dev/null @@ -1,13 +0,0 @@ -**`superduperdb.base.enums`** - -[Source code](https://github.com/SuperDuperDB/superduperdb/blob/main/superduperdb/base/enums.py) - -## `DBType` - -```python -DBType(self, - *args, - **kwds) -``` -DBType is an enumeration of the supported database types. - diff --git a/docs/content/api/base/logger.md b/docs/content/api/base/logger.md deleted file mode 100644 index c5f8f93d2a..0000000000 --- a/docs/content/api/base/logger.md +++ /dev/null @@ -1,24 +0,0 @@ -**`superduperdb.base.logger`** - -[Source code](https://github.com/SuperDuperDB/superduperdb/blob/main/superduperdb/base/logger.py) - -## `logging` - -```python -logging(self, - /, - *args, - **kwargs) -``` -Logging class to handle logging for the SuperDuperDB. - -## `Logging` - -```python -Logging(self, - /, - *args, - **kwargs) -``` -Logging class to handle logging for the SuperDuperDB. - diff --git a/docs/content/api/base/serializable.md b/docs/content/api/base/serializable.md deleted file mode 100644 index fb1a327a4d..0000000000 --- a/docs/content/api/base/serializable.md +++ /dev/null @@ -1,41 +0,0 @@ -**`superduperdb.base.serializable`** - -[Source code](https://github.com/SuperDuperDB/superduperdb/blob/main/superduperdb/base/serializable.py) - -## `Serializable` - -```python -Serializable(self) -> None -``` -Base class for serializable objects. - -This class is used to serialize and -deserialize objects to and from JSON + Artifact instances. - -## `Variable` - -```python -Variable(self, - value: Any, - setter_callback: dataclasses.InitVar[typing.Optional[typing.Callable]] = None) -> None -``` -| Parameter | Description | -|-----------|-------------| -| value | The name of the variable to be set at runtime. | -| setter_callback | A callback function that takes the value, datalayer and kwargs as input and returns the formatted variable. | - -Mechanism for allowing "free variables" in a serializable object. - -The idea is to allow a variable to be set at runtime, rather than -at object creation time. - -## `VariableError` - -```python -VariableError(self, - /, - *args, - **kwargs) -``` -Variable error. - diff --git a/docs/content/api/ext/auto.md b/docs/content/api/ext/auto.md deleted file mode 100644 index c1fe09f354..0000000000 --- a/docs/content/api/ext/auto.md +++ /dev/null @@ -1,4 +0,0 @@ -**`superduperdb.ext.auto`** - -[Source code](https://github.com/SuperDuperDB/superduperdb/blob/main/superduperdb/ext/auto.py) - diff --git a/docs/content/api/ext/transformers/model.md b/docs/content/api/ext/transformers/model.md index 8a543dface..7294de8c83 100644 --- a/docs/content/api/ext/transformers/model.md +++ b/docs/content/api/ext/transformers/model.md @@ -53,7 +53,7 @@ LLM(self, | adapter_id | adapter id, default is None Add a adapter to the base model for inference. | | model_kwargs | model kwargs, all the kwargs will pass to `transformers.AutoModelForCausalLM.from_pretrained` | | tokenizer_kwargs | tokenizer kwargs, all the kwargs will pass to `transformers.AutoTokenizer.from_pretrained` | -| prompt_template | prompt template, default is "{input}" | +| prompt_template | prompt template, default is `"{input}"` | LLM model based on `transformers` library. diff --git a/docs/content/api/misc/annotations.md b/docs/content/api/misc/annotations.md index 526ea0b359..d9b872cae4 100644 --- a/docs/content/api/misc/annotations.md +++ b/docs/content/api/misc/annotations.md @@ -33,7 +33,7 @@ This will result in a warning being emitted when the function is used. ```python component(*schema: Dict, - handle_integration: Callable = at 0x104aa3c40>) + handle_integration: Callable = at 0x10694fc40>) ``` | Parameter | Description | |-----------|-------------| diff --git a/docs/content/api/misc/colors.md b/docs/content/api/misc/colors.md deleted file mode 100644 index d8ee098954..0000000000 --- a/docs/content/api/misc/colors.md +++ /dev/null @@ -1,14 +0,0 @@ -**`superduperdb.misc.colors`** - -[Source code](https://github.com/SuperDuperDB/superduperdb/blob/main/superduperdb/misc/colors.py) - -## `Colors` - -```python -Colors(self, - /, - *args, - **kwargs) -``` -Colors list for terminal output. - diff --git a/docs/content/api/misc/files.md b/docs/content/api/misc/files.md deleted file mode 100644 index 59b82837de..0000000000 --- a/docs/content/api/misc/files.md +++ /dev/null @@ -1,48 +0,0 @@ -**`superduperdb.misc.files`** - -[Source code](https://github.com/SuperDuperDB/superduperdb/blob/main/superduperdb/misc/files.py) - -## `load_uris` - -```python -load_uris(r: dict, - datatypes: Dict, - root: Optional[str] = None, - raises: bool = False) -``` -| Parameter | Description | -|-----------|-------------| -| r | The dict to load the bytes into | -| datatypes | The datatypes to use for encoding | -| root | The root directory to load the bytes from | -| raises | Whether to raise an error if the file is not found | - -Load ``"bytes"`` into ``"_content"`` from ``"uri"`` inside ``r``. - -```python -with open('/tmp/test.txt', 'wb') as f: - _ = f.write(bytes('test', 'utf-8')) -r = {"_content": {"uri": "file://test.txt"}} -load_uris(r, '/tmp') -r -# {'_content': {'uri': 'file://test.txt', 'bytes': b'test'}} -``` - -## `get_file_from_uri` - -```python -get_file_from_uri(uri) -``` -| Parameter | Description | -|-----------|-------------| -| uri | The uri to get the file from | - -Get file name from uri. - -```python -_get_file('file://test.txt') -# 'test.txt' -_get_file('http://test.txt') -# '414388bd5644669b8a92e45a96318890f6e8de54' -``` - diff --git a/docs/content/api/misc/serialization.md b/docs/content/api/misc/serialization.md index 49dc5054ab..7b04c58198 100644 --- a/docs/content/api/misc/serialization.md +++ b/docs/content/api/misc/serialization.md @@ -7,7 +7,7 @@ ```python asdict(obj, *, - copy_method=) -> Dict[str, + copy_method=) -> Dict[str, Any] ``` | Parameter | Description | diff --git a/superduperdb/ext/transformers/model.py b/superduperdb/ext/transformers/model.py index 0e4a5aaeb9..cebd87f817 100644 --- a/superduperdb/ext/transformers/model.py +++ b/superduperdb/ext/transformers/model.py @@ -249,7 +249,7 @@ class LLM(BaseLLM, _Fittable): all the kwargs will pass to `transformers.AutoModelForCausalLM.from_pretrained` :param tokenizer_kwargs: tokenizer kwargs, all the kwargs will pass to `transformers.AutoTokenizer.from_pretrained` - :param prompt_template: prompt template, default is "{input}" + :param prompt_template: prompt template, default is `"{input}"` :param prompt_func: prompt function, default is None All the `model_kwargs` will pass to @@ -298,11 +298,11 @@ def from_pretrained( """A new function to create a LLM model from from_pretrained function. Allow the user to directly replace: - AutoModelForCausalLM.from_pretrained -> LLM.from_pretrained + `AutoModelForCausalLM.from_pretrained` -> `LLM.from_pretrained` :param model_name_or_path: model name or path :param identifier: model identifier - :param prompt_template: prompt template, default is "{input}" + :param prompt_template: prompt template, default is `"{input}"` :param prompt_func: prompt function, default is None :param predict_kwargs: predict kwargs, default is None :param kwargs: additional keyword arguments, all the kwargs will pass to `LLM`