Skip to content

Commit

Permalink
Merge pull request #8026 from garazdawi/lukas/otp/convert-docs-to-mar…
Browse files Browse the repository at this point in the history 
…kdown/OTP-18955

Change documentation to use ExDoc and EEP-59 style documentation
  • Loading branch information
@garazdawi
garazdawi committed Jan 31, 2024
2 parents 8864369 + 36dddd6 commit 78342b2
Show file tree
Hide file tree
Showing 3,258 changed files with 325,239 additions and 487,186 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
2 changes: 1 addition & 1 deletion .github/dockerfiles/Dockerfile.debian-base
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ RUN apt-get update && apt-get -y upgrade && apt-get install -y build-essential &
sed -i "s:deb http:deb [arch=$BUILD_ARCH,$HOST_ARCH] http:g" /etc/apt/sources.list; \
fi && \
apt-get update && \
apt-get install -y build-essential m4 autoconf fop xsltproc default-jdk libxml2-utils \
apt-get install -y build-essential m4 autoconf default-jdk \
$INSTALL_LIBS && \
if [ "$HOST_TRIP" != "$BUILD_TRIP" ]; then \
apt-get install -y -f \
Expand Down
29 changes: 27 additions & 2 deletions .github/dockerfiles/Dockerfile.ubuntu-base
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ ENV LANG=C.UTF-8 LC_ALL=C.UTF-8

## Install build tools
RUN apt-get update && apt-get -y upgrade && \
apt-get install -y build-essential m4 autoconf fop xsltproc \
default-jdk libxml2-utils flex pkg-config locales tzdata sudo ${INSTALL_LIBS} && \
apt-get install -y build-essential m4 autoconf \
default-jdk flex pkg-config locales tzdata sudo ${INSTALL_LIBS} && \
sed -i 's@# en_US.UTF-8@en_US.UTF-8@g' /etc/locale.gen && locale-gen && \
update-alternatives --set wx-config /usr/lib/x86_64-linux-gnu/wx/config/gtk3-unicode-3.0

Expand Down Expand Up @@ -56,6 +56,8 @@ RUN apt-get install -y git curl && \
VSN=$(kerl list releases | grep "^$release" | tail -1); \
if [ $release = $LATEST ]; then \
echo "/usr/local/lib/erlang-${VSN}/bin" > /home/${USER}/LATEST; \
echo "${VSN}" > /home/${USER}/LATEST_VSN; \
echo "${release}" > /home/${USER}/LATEST_RELEASE; \
fi && \
kerl build ${VSN} ${VSN} && \
kerl install ${VSN} /usr/local/lib/erlang-${VSN}; \
Expand Down Expand Up @@ -104,6 +106,29 @@ COPY --chown=${USER}:${GROUP} dockerfiles/init.sh /buildroot/

WORKDIR /buildroot/

## Build and install elixir, hex and ex_doc
ENV ELIXIR_VSN=latest \
HEX_VSN=latest \
EX_DOC_VSN=latest
RUN export PATH="$(cat /home/${USER}/LATEST):${PATH}" && \
set -x && \
mkdir /buildroot/elixir && cd /buildroot/elixir && \
curl -s https://api.github.com/repos/elixir-lang/elixir/releases/${ELIXIR_VSN} | \
jq '.tarball_url' | xargs curl -L > elixir.tar.gz && \
tar xzf elixir.tar.gz --strip-components=1 && \
make && sudo make install && \
mkdir /buildroot/hex && cd /buildroot/hex && \
curl -s https://api.github.com/repos/hexpm/hex/releases/${HEX_VSN} | \
jq '.tarball_url' | xargs curl -L > hex.tar.gz && \
tar xzf hex.tar.gz --strip-components=1 && \
mix install && \
mkdir /buildroot/ex_doc && cd /buildroot/ex_doc && \
curl -s https://api.github.com/repos/elixir-lang/ex_doc/releases/${EX_DOC_VSN} | \
jq '.tarball_url' | xargs curl -L > ex_doc.tar.gz && \
tar xzf ex_doc.tar.gz --strip-components=1 && \
mix deps.get && MIX_ENV=prod mix escript.build && \
sudo cp ex_doc /usr/local/bin/

## Install test tools rebar3, proper and jsx
RUN export PATH="$(cat /home/${USER}/LATEST):${PATH}" && \
latest () { \
Expand Down
2 changes: 1 addition & 1 deletion .github/scripts/restore-from-prebuilt.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ if [ -n "${ARCHIVE}" ]; then

echo "::group::{Run ${i}: docs}"
### If any of the doc generating tools change, we need to re-compile the docs
if grep "lib/\(xmerl\|erl_docgen\|edoc\)" "${CHANGES}"; then
if grep "lib/\(xmerl\|edoc\)" "${CHANGES}"; then
echo "Deleting all docs as documentation tools have changed" >&2
rm -rf "${CACHE_DIR}"/lib/*/doc/ "${CACHE_DIR}/erts/doc/" "${CACHE_DIR}/system/"
fi
Expand Down
29 changes: 10 additions & 19 deletions .github/workflows/main.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,34 +396,29 @@ jobs:
## Build all the documentation
- name: Build documentation
run: |
docker build -t otp - <<EOF
if [ "${{ github.event_name }}" = "pull_request" ]; then
BASE_URL="${{ github.event.pull_request.head.repo.full_name }}/blob/${{ github.event.pull_request.head.ref }}/"
else
BASE_URL="${{ github.repository }}/blob/${{ github.ref_name }}/"
fi
docker build --build-arg BASE_URL="$BASE_URL" -t otp - <<EOF
FROM otp
ENV BASE_URL=$BASE_URL
RUN make release docs release_docs && sudo make install-docs
EOF
- name: Release docs to publish
run: |
docker run -v $PWD/:/github otp \
"make release_docs DOC_TARGETS='man html pdf' RELEASE_ROOT=/github/docs"
"make release_docs DOC_TARGETS='html' RELEASE_ROOT=/github/docs"
sudo chown -R `whoami` docs
cd docs
tar czf ../otp_doc_man.tar.gz man
rm -rf man
tar czf ../otp_doc_html.tar.gz *
cd docs && tar czf ../otp_doc_html.tar.gz *
- name: Upload html documentation archive
uses: actions/upload-artifact@v4.3.0
with:
name: otp_doc_html
path: otp_doc_html.tar.gz
- name: Upload man documentation archive
uses: actions/upload-artifact@v4.3.0
with:
name: otp_doc_man
path: otp_doc_man.tar.gz
## Documentation checks
- name: Run xmllint
run: docker run otp "make xmllint"
- name: Run html link check
run: docker run -v $PWD/:/github otp "/github/scripts/otp_html_check /github/docs doc/index.html"
run: docker run -v $PWD/:/github otp "cd /github/docs && /github/scripts/otp_check_html_links.exs"

static:
name: Run static analysis
Expand Down Expand Up @@ -585,10 +580,6 @@ jobs:
uses: actions/download-artifact@v4.1.1
with:
name: otp_doc_html
- name: Download man docs
uses: actions/download-artifact@v4.1.1
with:
name: otp_doc_man

## We add the correct version name into the file names
## and create the hash files for all assets
Expand Down
29 changes: 3 additions & 26 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ JAVADOC-GENERATED
/Makefile

/release
/doc

/make/output.mk
/make/emd2exml
Expand All @@ -164,20 +165,7 @@ JAVADOC-GENERATED
/lib/*/SKIP
/lib/SKIP-APPLICATIONS

/lib/*/doc/html/*.html
/lib/*/doc/html/*.css
/lib/*/doc/html/js
/lib/*/doc/html/*.gif
/lib/*/doc/html/*.jpg
/lib/*/doc/html/*.png
/lib/*/doc/html/*.eix
/lib/*/doc/man[0-9]/*.[0-9]
/lib/*/doc/pdf/*.fo
/lib/*/doc/pdf/*.pdf
/lib/*/doc/src/*.fo
/lib/*/doc/xml/*.xml
/lib/*/doc/xml/*.ent
/lib/*/doc/chunks/*.chunk
/lib/*/doc/html

/lib/config.log
/lib/config.status
Expand Down Expand Up @@ -260,18 +248,7 @@ JAVADOC-GENERATED
# erts

/erts/info
/erts/doc/html/*.html
/erts/doc/html/*.gif
/erts/doc/html/*.jpg
/erts/doc/html/*.eix
/erts/doc/pdf/*.fo
/erts/doc/pdf/*.pdf
/erts/doc/src/*.fo
/erts/doc/xml/*.xml
/erts/doc/xml/figures/*.png
/erts/doc/xml/figures/*.svg
/erts/doc/chunks/*.chunk
/erts/doc/man[0-9]/*.[0-9]
/erts/doc/html/
/erts/doc/CONF_INFO

# et
Expand Down
5 changes: 2 additions & 3 deletions CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ There are exceptions, for example corrections to bugs that have been introduced
* TIP: Write the test case **before** fixing the bug so that you can know that it catches the bug.

* For applications without a test suite in the git repository, it would be appreciated if you provide a
small code sample in the commit message or email a module that will provoke the failure.
small code sample in the commit message or attach a module to the PR that will provoke the failure.

### Adding a new feature

Expand Down Expand Up @@ -100,8 +100,7 @@ a discussion on the mailing list.

* Make sure existing test cases don't fail. It is not necessary to run all tests (that would take many hours),
but you should at least run the tests for the application you have changed.
* Make sure the documentation builds and is according to the dtd. eg. `make xmllint` or `cd lib/stdlib/ && make xmllint`
* Make sure no new dialyzer warnings have been added. eg. `make dialyzer` or `cd lib/stdlib/ && make dialyzer`
* Make sure all static checks pass by calling `./otp_build check`. Call `./otp_build check --help` for details on what `./otp_build check` does.
* Make sure that github actions passes, if you go to https://github.com/$YOUR_GITHUB_USER/otp/actions you can enable github actions builds for you otp fork.

See the [Testing](https://github.com/erlang/otp/blob/master/HOWTO/TESTING.md) and
Expand Down
110 changes: 31 additions & 79 deletions HOWTO/DEVELOPMENT.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ make # Rebuid application
make test # Run application tests
make dialyzer # Run dialyzer
make docs # Build the docs
make xmllint # Run xmllint on the docs
```

Then enable [Github Actions](#github-actions) and push the changes to your fork
Expand Down Expand Up @@ -189,13 +188,6 @@ make stdlib_test ARGS="-suite lists_SUITE -case member"
See [ct_run](https://www.erlang.org/doc/man/ct_run.html#) for a list of all options
that you can pass to ARGS.

You can run static analysis test:

```bash
make dialyzer # Checks all of Erlang/OTP source code
make xmllint # Checks all documentation for xmllint errors
```

Most of the above targets also works for a "phony" target called `emulator` that
represents erts and all its tools. So you can do this:

Expand All @@ -213,6 +205,14 @@ for all process you would do this:
ERL_ARGS="+hmqd off_heap" make emulator_test
```

You can run static analysis test:

```bash
./otp_build check
```

See `./otp_build check --help` for details on how to use it and what it does.

### Build and test a specific application

You can also build the application from within itself. Like this:
Expand All @@ -230,17 +230,9 @@ make test ARGS="-suite lists_SUITE" # run the lists_SUITE tests
make dialyzer # run dialyzer for this application
make docs # build all docs for this application
make docs DOC_TARGETS="html" # build html docs for this application
make xmllint # run xmllint on the docs for this application
```

If you want to view what the documentation looks like for only your application
you can do this:

```bash
(cd doc/src && make local_docs)
```

and then view `doc/html/index.html`.
Open `doc/html/index.html` in your favorite web browser to view the documentation.

### Preloaded and Primary Bootstrap

Expand Down Expand Up @@ -382,51 +374,44 @@ the emulator start script. Instead you need to use the approach described in
From the top level of Erlang/OTP you can run:

```bash
make xmllint
make dialyzer
make format-check
./otp_build check
```

This will check that the documentation is correct and that there are no
dialyzer errors. See also [Validating documentation](#validating-documentation)
for more details.
This will check that the documentation is correct, that there are no
dialyzer errors and many more things.

## Running test cases

There is a detailed description about how to run tests in [TESTING.md](TESTING.md).

## Writing and building documentation

Most of the Erlang/OTP documentation is written in XML files located in
`lib/$APPLICATION_NAME/doc/src`. The format of the XML is described in the
[ErlDocgen User's Guide](https://www.erlang.org/doc/apps/erl_docgen/users_guide.html).
Most of the Erlang/OTP documentation is written using markdown, either directly
in the module implementing the functionality, or as `.md` files located in
`lib/$APPLICATION_NAME/doc`. For more details about how to write documentation see [Documentation](`../system/doc/reference_manual/documentation.md`) and [Documentation HOWTO](DOCUMENTATION.md).

There is also some documentation that is written using [edoc](https://www.erlang.org/doc/man/edoc.html).

To view the documentation the simplest way is to release it. *NOTE*: The Erlang/OTP
In order to build the documentation you need to have [ex_doc](https://hexdocs.pm/ex_doc/readme.html)
installed and available in your path. The simplest way to do that is to download
the [escript for the latest release](https://github.com/elixir-lang/ex_doc/releases/latest) available on github.

To view the documentation you need to build it. *NOTE*: The Erlang/OTP
repository needs to have been [built](#building-and-testing) before you can build
the documentation.

```bash
make release_docs
make docs
```

and then you can view `release/*/doc/index.html` in your favourite browser and
and then you can view `doc/index.html` in your favourite browser and
make sure that it looks nice.

This takes a while though and to speed up the edit-view cycle you can either
limit what parts of the documentation is built using `DOC_TARGETS`. For example:
This takes a while though and to speed up the edit-view cycle you can build the
docs only for a single application. For example:

```bash
make release_docs DOC_TARGETS=html
```

The different `DOC_TARGETS` built are `html`, `man`, `pdf` and `chunks`.

You can also build the docs only for a single application. For example:

```bash
cd lib/stdlib/doc/src && make local_docs DOC_TARGETS=html
cd lib/stdlib/doc/src && make docs
```

and then view the results at `lib/stdlib/doc/html/index.html`.
Expand All @@ -435,51 +420,18 @@ and then view the results at `lib/stdlib/doc/html/index.html`.

In order to make sure that the documentation is correct you need to also
validate it. Just building the documentation does not mean that it is
correct. There are two steps that you need to do in order to validate
the docs.
correct.

First run the `xmllint`. This makes sure that the xml follows the
[Erlang/OTP dtd](https://www.erlang.org/doc/apps/erl_docgen/overview.html)
and does some extra checks to make sure all links are correct.
You need to make sure that there are no broken links in the documentation.
This is done by running `./otp_build check`.

You run the xmllint like this:

```bash
make xmllint # Run it at the top level
cd lib/stdlib && make xmllint # Run it for only a single application
```

When the xml has been verified you also need to make sure that there
are no broken links in the documentation. This is done by running
[`otp_html_check`](https://github.com/erlang/otp/blob/master/scripts/otp_html_check).

You run `otp_html_check` like this:

```bash
make release_docs DOC_TARGETS="html pdf" # First we need to release the pdf+html docs
$ERL_TOP/scripts/otp_html_check $(pwd)/release/$($ERL_TOP/erts/autoconf/config.guess) doc/index.html
```

The output of `otp_html_check` will print a list of broken links and anchors, for example:
The output of `./otp_build check` will print a list of broken links and anchors, for example:

```text
**** Files not used (that I can see)
....
**** Broken links
Broken Link: /lib/kernel-8.3.1/doc/html/inet.html -> "/lib/kernel-8.3.1/doc/html/files.html"
**** References to missing anchors
Missing Anchor: "/lib/kernel-8.3.1/doc/html/files.html#native_name_encoding-0" from /lib/kernel-8.3.1/doc/html/inet.html
Missing Anchor: "/lib/kernel-8.3.1/doc/html/inet.html#type-ip_addres" from /lib/kernel-8.3.1/doc/html/inet.html
lib/kernel/doc/html/eep48_chapter.html: Found duplicate anchor see-also
lib/kernel/doc/html/eep48_chapter.html: could not find lib/stdlib/doc/html/shell_docs.html#contents, should it be #content?
```

The `Files not used` section is mostly used to debug the documentation build
process, so it can be ignored most of the time. The other to sections should
however be empty for the documentation to be correct.

All this validation is also done by [Github Actions](#github-actions).

## Github Actions
Expand Down

0 comments on commit 78342b2

Please sign in to comment.