Fix make docs_build and related scripts

.github/CONTRIBUTING.md

@@ -224,30 +224,38 @@ When you run `poetry install`, the `langchain` package is installed as editable
 
 ### Contribute Documentation
 
-Docs are largely autogenerated by [sphinx](https://www.sphinx-doc.org/en/master/) from the code.
+The docs directory contains Documentation and API Reference.
 
+Documentation is built using [Docusaurus 2](https://docusaurus.io/).
+
+API Reference are largely autogenerated by [sphinx](https://www.sphinx-doc.org/en/master/) from the code.
 For that reason, we ask that you add good documentation to all classes and methods.
 
 Similar to linting, we recognize documentation can be annoying. If you do not want to do it, please contact a project maintainer, and they can help you with it. We do not want this to be a blocker for good code getting contributed.
 
 ### Build Documentation Locally
 
+In the following commands, the prefix `api_` indicates that those are operations for the API Reference.
+
 Before building the documentation, it is always a good idea to clean the build directory:
 
 ```bash
 make docs_clean
+make api_docs_clean
 ```
 
-Next, you can run the linkchecker to make sure all links are valid:
+Next, you can build the documentation as outlined below:
 
 ```bash
-make docs_linkcheck
+make docs_build
+make api_docs_build
 ```
 
-Finally, you can build the documentation as outlined below:
+Finally, you can run the linkchecker to make sure all links are valid:
 
 ```bash
-make docs_build
+make docs_linkcheck
+make api_docs_linkcheck
 ```
 
 ## 🏭 Release Process

.gitignore

@@ -161,7 +161,12 @@ docs/node_modules/
 docs/.docusaurus/
 docs/.cache-loader/
 docs/_dist
+docs/api_reference/api_reference.rst
 docs/api_reference/_build
+docs/api_reference/*/
+!docs/api_reference/_static/
+!docs/api_reference/templates/
+!docs/api_reference/themes/
 docs/docs_skeleton/build
 docs/docs_skeleton/node_modules
 docs/docs_skeleton/yarn.lock

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all clean format lint test tests test_watch integration_tests docker_tests help extended_tests
+.PHONY: all clean docs_build docs_clean docs_linkcheck api_docs_build api_docs_clean api_docs_linkcheck format lint test tests test_watch integration_tests docker_tests help extended_tests
 
 # Default target executed when no arguments are given to make.
 all: help
@@ -14,6 +14,33 @@ coverage:
 		--cov-report xml \
 		--cov-report term-missing:skip-covered
 
+######################
+# DOCUMENTATION
+######################
+
+clean: docs_clean api_docs_clean
+
+
+docs_build:
+	docs/.local_build.sh
+
+docs_clean:
+	rm -r docs/_dist
+
+docs_linkcheck:
+	poetry run linkchecker docs/_dist/docs_skeleton/ --ignore-url node_modules
+
+api_docs_build:
+	poetry run python docs/api_reference/create_api_rst.py
+	cd docs/api_reference && poetry run make html
+
+api_docs_clean:
+	rm -f docs/api_reference/api_reference.rst
+	cd docs/api_reference && poetry run make clean
+
+api_docs_linkcheck:
+	poetry run linkchecker docs/api_reference/_build/html/index.html
+
 # Define a variable for the test file path.
 TEST_FILE ?= tests/unit_tests/
 
@@ -36,22 +63,6 @@ docker_tests:
 	docker build -t my-langchain-image:test .
 	docker run --rm my-langchain-image:test
 
-######################
-# DOCUMENTATION
-######################
-
-docs_compile:
-	poetry run nbdoc_build --srcdir $(srcdir)
-
-docs_build:
-	cd docs && poetry run make html
-
-docs_clean:
-	cd docs && poetry run make clean
-
-docs_linkcheck:
-	poetry run linkchecker docs/_build/html/index.html
-
 ######################
 # LINTING AND FORMATTING
 ######################

docs/.local_build.sh

@@ -1,10 +1,15 @@
-mkdir _dist
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace
+
+SCRIPT_DIR="$(cd "$(dirname "$0")"; pwd)"
+cd "${SCRIPT_DIR}"
+
+mkdir -p _dist/docs_skeleton
 cp -r {docs_skeleton,snippets} _dist
-mkdir -p _dist/docs_skeleton/static/api_reference
-cd api_reference
-poetry run make html
-cp -r _build/* ../_dist/docs_skeleton/static/api_reference
-cd ..
 cp -r extras/* _dist/docs_skeleton/docs
 cd _dist/docs_skeleton
 poetry run nbdoc_build