From cb4b9c6b726f3cfcba7ab235ee6146ded3a450f7 Mon Sep 17 00:00:00 2001 From: Benj Fassbind Date: Tue, 16 Aug 2022 14:45:14 +0200 Subject: [PATCH] feat: add project api This change massively improves the usability while doing local development. It also fixes #40. The local development now also proxies the `/api` and `/doc` endpoints from the backend (port 5000) through the frontend (port 8080) making CORS handling simpler during development. --- Dockerfile | 2 - README.md | 36 ++++++++++++--- docat/docat/app.py | 56 +++++++++++++++++++++-- docat/docat/nginx/default | 2 - docat/docat/templates/nginx-doc.conf | 5 -- docat/docat/utils.py | 23 ---------- docat/tests/conftest.py | 5 +- docat/tests/test_project.py | 45 ++++++++++++++++++ docat/tests/test_upload.py | 8 ++-- docat/tests/test_utils.py | 46 +++---------------- web/README.md | 1 - web/src/components/ProjectOverview.vue | 4 +- web/src/pages/Delete.vue | 2 +- web/src/pages/Docs.vue | 10 ++-- web/src/repositories/ProjectRepository.js | 23 +++++----- web/tests/unit/project-repository.spec.js | 32 +++++++------ web/vue.config.js | 13 ++++++ 17 files changed, 189 insertions(+), 124 deletions(-) delete mode 100644 docat/docat/templates/nginx-doc.conf create mode 100644 docat/tests/test_project.py diff --git a/Dockerfile b/Dockerfile index fecf3d03..c32f30e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,9 +42,7 @@ RUN apk update && \ apk add nginx dumb-init && \ rm -rf /var/cache/apk/* -RUN mkdir -p /etc/nginx/locations.d RUN mkdir -p /var/docat/doc -RUN chown -R nginx /var/docat /etc/nginx/locations.d # install the application RUN mkdir -p /var/www/html diff --git a/README.md b/README.md index 30c84c90..811eff66 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ mkdir -p docat-run/db && touch docat-run/db/db.json docker run \ --detach \ --volume $PWD/docat-run/doc:/var/docat/doc/ \ - --volume $PWD/docat-run/locations:/etc/nginx/locations.d/ \ --volume $PWD/docat-run/db/db.json:/app/docat/db.json \ --publish 8000:80 \ ghcr.io/docat-org/docat @@ -32,7 +31,6 @@ mkdir -p docat-run/db && touch docat-run/db/db.json docker run \ --detach \ --volume $PWD/docat-run/doc:/var/docat/doc/ \ - --volume $PWD/docat-run/locations:/etc/nginx/locations.d/ \ --volume $PWD/docat-run/db:/var/docat/db/ \ --env DOCAT_DB_PATH=/var/docat/db/db.json --publish 8000:80 \ @@ -43,10 +41,35 @@ Go to [localhost:8000](http://localhost:8000) to view your docat instance: ![docat screenshot](doc/assets/docat-screenshot.png) -If you want to run the application different than in a docker container, look at the +### Local Development + +For local development, first configure and start the backend (inside the `docat/` folder): + +```sh +# create a folder for local development (uploading docs) +DEV_DOC_PATH="$(mktemp -d)" + +# install dependencies +poetry install + +# run the local development version +DOCAT_SERVE_FILES=1 DOCAT_DOC_PATH="$DEV_DOC_PATH" poetry run python -m docat +``` + +After this you need to start the frontend (inside the `web/` folder): + +```sh +# install dependencies +yarn install --frozen-lockfile + +# run the web app +yarn serve +``` + +For more advanced options, have a look at the [backend](docat/README.md) and [web](web/README.md) docs. -### Push documentation to docat +### Push Documentation to docat The preferred way to push documentation to a docat server is using the [docatl](https://github.com/docat-org/docatl) command line application: @@ -57,7 +80,7 @@ docatl push --host http://localhost:8000 /path/to/your/docs PROJECT VERSION There are also docker images available for CI systems. -#### Using standard UNIX command line tools +#### Using Standard UNIX Command Line Tools If you have static html documentation or use something like [mkdocs](https://www.mkdocs.org/), [sphinx](http://www.sphinx-doc.org/en/master/), ... @@ -77,8 +100,7 @@ When you have multiple versions you may want to tag some version as **latest**: curl -X PUT http://localhost:8000/api/PROJECT/VERSION/tags/latest ``` - -## Advanced config.json +## Advanced `config.json` It is possible to configure some things after the fact. diff --git a/docat/docat/app.py b/docat/docat/app.py index 0864c172..08689e71 100644 --- a/docat/docat/app.py +++ b/docat/docat/app.py @@ -20,7 +20,7 @@ from starlette.responses import JSONResponse from tinydb import Query, TinyDB -from docat.utils import DB_PATH, UPLOAD_FOLDER, calculate_token, create_nginx_config, create_symlink, extract_archive, remove_docs +from docat.utils import DB_PATH, UPLOAD_FOLDER, calculate_token, create_symlink, extract_archive, remove_docs #: Holds the FastAPI application app = FastAPI( @@ -56,6 +56,57 @@ class ClaimResponse(ApiResponse): token: str +class ProjectsResponse(BaseModel): + projects: list[str] + + +class ProjectVersion(BaseModel): + name: str + tags: list[str] + + +class ProjectDetailResponse(BaseModel): + name: str + versions: list[ProjectVersion] + + +@app.get("/api/projects", response_model=ProjectsResponse, status_code=status.HTTP_200_OK) +def get_projects(): + if not DOCAT_UPLOAD_FOLDER.exists(): + return ProjectsResponse(projects=[]) + return ProjectsResponse(projects=[str(x.relative_to(DOCAT_UPLOAD_FOLDER)) for x in DOCAT_UPLOAD_FOLDER.iterdir() if x.is_dir()]) + + +@app.get( + "/api/projects/{project}", + response_model=ProjectDetailResponse, + status_code=status.HTTP_200_OK, + responses={status.HTTP_404_NOT_FOUND: {"model": ApiResponse}}, +) +def get_project(project): + docs_folder = DOCAT_UPLOAD_FOLDER / project + if not docs_folder.exists(): + return JSONResponse(status_code=status.HTTP_404_NOT_FOUND, content={"message": f"Project {project} does not exist"}) + + tags = [x for x in docs_folder.iterdir() if x.is_dir() and x.is_symlink()] + + return ProjectDetailResponse( + name=project, + versions=sorted( + [ + ProjectVersion( + name=str(x.relative_to(docs_folder)), + tags=[str(t.relative_to(docs_folder)) for t in tags if t.resolve() == x], + ) + for x in docs_folder.iterdir() + if x.is_dir() and not x.is_symlink() + ], + key=lambda k: k.name, + reverse=True, + ), + ) + + @app.post("/api/{project}/{version}", response_model=ApiResponse, status_code=status.HTTP_201_CREATED) def upload( project: str, @@ -86,7 +137,6 @@ def upload( shutil.copyfileobj(file.file, buffer) extract_archive(target_file, base_path) - create_nginx_config(project, project_base_path) return ApiResponse(message="File successfully uploaded") @@ -154,4 +204,4 @@ def check_token_for_project(db, token, project) -> TokenStatus: # serve_local_docs for local testing without a nginx if os.environ.get("DOCAT_SERVE_FILES"): - app.mount("/doc", StaticFiles(directory=DOCAT_UPLOAD_FOLDER), name="docs") + app.mount("/doc", StaticFiles(directory=DOCAT_UPLOAD_FOLDER, html=True), name="docs") diff --git a/docat/docat/nginx/default b/docat/docat/nginx/default index 1a961edf..712cb286 100644 --- a/docat/docat/nginx/default +++ b/docat/docat/nginx/default @@ -15,8 +15,6 @@ server { location /doc { root /var/docat; - autoindex on; - autoindex_format json; } location /api { diff --git a/docat/docat/templates/nginx-doc.conf b/docat/docat/templates/nginx-doc.conf deleted file mode 100644 index d9e3a4b4..00000000 --- a/docat/docat/templates/nginx-doc.conf +++ /dev/null @@ -1,5 +0,0 @@ -location /doc/{{ project }} { - alias {{ dir_path }}; - autoindex on; - autoindex_format json; -} diff --git a/docat/docat/utils.py b/docat/docat/utils.py index 20e009b5..3143113a 100644 --- a/docat/docat/utils.py +++ b/docat/docat/utils.py @@ -4,12 +4,9 @@ import hashlib import os import shutil -import subprocess from pathlib import Path from zipfile import ZipFile -from jinja2 import Template - NGINX_CONFIG_PATH = Path("/etc/nginx/locations.d") UPLOAD_FOLDER = Path("/var/docat/doc") DB_PATH = "db.json" @@ -33,26 +30,6 @@ def create_symlink(source, destination): return False -def create_nginx_config(project, project_base_path): - """ - Creates an Nginx configuration for an uploaded project - version. - - Args: - project (str): name of the project - project_base_path (pathlib.Path): base path of the project - """ - nginx_config = NGINX_CONFIG_PATH / f"{project}-doc.conf" - if not nginx_config.exists(): - out_parsed_template = Template((Path(__file__).parent / "templates" / "nginx-doc.conf").read_text()).render( - project=project, dir_path=str(project_base_path) - ) - with nginx_config.open("w") as f: - f.write(out_parsed_template) - - subprocess.run(["nginx", "-s", "reload"]) - - def extract_archive(target_file, destination): """ Extracts the given archive to the directory diff --git a/docat/tests/conftest.py b/docat/tests/conftest.py index cbd508d7..404c9983 100644 --- a/docat/tests/conftest.py +++ b/docat/tests/conftest.py @@ -31,19 +31,16 @@ def client_with_claimed_project(client): @pytest.fixture def temp_project_version(tmp_path): docs = tmp_path / "doc" - config = tmp_path / "location.d" docs.mkdir() - config.mkdir() def __create(project, version): - (config / f"{project}-doc.conf").touch() version_docs = docs / project / version version_docs.mkdir(parents=True) (version_docs / "index.html").touch() create_symlink(version_docs, docs / project / "latest") - return docs, config + return docs yield __create diff --git a/docat/tests/test_project.py b/docat/tests/test_project.py new file mode 100644 index 00000000..67aa3111 --- /dev/null +++ b/docat/tests/test_project.py @@ -0,0 +1,45 @@ +from unittest.mock import patch + +from fastapi.testclient import TestClient + +from docat.app import app + +client = TestClient(app) + + +def test_project_api(temp_project_version): + project = "project" + docs = temp_project_version(project, "1.0") + + with patch("docat.app.DOCAT_UPLOAD_FOLDER", docs): + response = client.get("/api/projects") + + assert response.ok + assert response.json() == {"projects": ["project"]} + + +def test_project_api_without_any_projects(): + response = client.get("/api/projects") + + assert response.ok + assert response.json() == {"projects": []} + + +def test_project_details_api(temp_project_version): + project = "project" + docs = temp_project_version(project, "1.0") + symlink_to_latest = docs / project / "latest" + assert symlink_to_latest.is_symlink() + + with patch("docat.app.DOCAT_UPLOAD_FOLDER", docs): + response = client.get(f"/api/projects/{project}") + + assert response.ok + assert response.json() == {"name": "project", "versions": [{"name": "1.0", "tags": ["latest"]}]} + + +def test_project_details_api_with_a_project_that_does_not_exist(): + response = client.get("/api/projects/i-do-not-exist") + + assert not response.ok + assert response.json() == {"message": "Project i-do-not-exist does not exist"} diff --git a/docat/tests/test_upload.py b/docat/tests/test_upload.py index d17b5f3e..e12556e7 100644 --- a/docat/tests/test_upload.py +++ b/docat/tests/test_upload.py @@ -3,7 +3,7 @@ def test_successfully_upload(client): - with patch("docat.app.remove_docs"), patch("docat.app.create_nginx_config"): + with patch("docat.app.remove_docs"): response = client.post("/api/some-project/1.0.0", files={"file": ("index.html", io.BytesIO(b"

Hello World

"), "plain/text")}) response_data = response.json() @@ -12,7 +12,7 @@ def test_successfully_upload(client): def test_successfully_override(client_with_claimed_project): - with patch("docat.app.remove_docs") as remove_mock, patch("docat.app.create_nginx_config"): + with patch("docat.app.remove_docs") as remove_mock: response = client_with_claimed_project.post( "/api/some-project/1.0.0", files={"file": ("index.html", io.BytesIO(b"

Hello World

"), "plain/text")} ) @@ -31,7 +31,7 @@ def test_successfully_override(client_with_claimed_project): def test_tags_are_not_overwritten_without_api_key(client_with_claimed_project): - with patch("docat.app.remove_docs") as remove_mock, patch("docat.app.create_nginx_config"): + with patch("docat.app.remove_docs") as remove_mock: response = client_with_claimed_project.post( "/api/some-project/1.0.0", files={"file": ("index.html", io.BytesIO(b"

Hello World

"), "plain/text")} ) @@ -51,7 +51,7 @@ def test_tags_are_not_overwritten_without_api_key(client_with_claimed_project): def test_fails_with_invalid_token(client_with_claimed_project): - with patch("docat.app.remove_docs") as remove_mock, patch("docat.app.create_nginx_config"): + with patch("docat.app.remove_docs") as remove_mock: response = client_with_claimed_project.post( "/api/some-project/1.0.0", files={"file": ("index.html", io.BytesIO(b"

Hello World

"), "plain/text")} ) diff --git a/docat/tests/test_utils.py b/docat/tests/test_utils.py index 339cbbaa..2ae3bd2b 100644 --- a/docat/tests/test_utils.py +++ b/docat/tests/test_utils.py @@ -1,7 +1,7 @@ from pathlib import Path -from unittest.mock import MagicMock, mock_open, patch +from unittest.mock import MagicMock, patch -from docat.utils import create_nginx_config, create_symlink, extract_archive, remove_docs +from docat.utils import create_symlink, extract_archive, remove_docs def test_symlink_creation(): @@ -54,38 +54,6 @@ def test_symlink_creation_do_not_overwrite_destination(): destination.symlink_to.assert_not_called() -def test_create_nginx_config(): - """ - Tests the creation of the nginx config - """ - jinja_template_mock = "{{ project }}:{{ dir_path }}" - with patch.object(Path, "read_text", return_value=jinja_template_mock), patch.object(Path, "exists") as mock_exists, patch.object( - Path, "open", mock_open() - ) as mock_config, patch("subprocess.run"): - mock_exists.return_value = False - - create_nginx_config("awesome-project", "/some/dir") - - mock_config.assert_called_once_with("w") - mock_config().write.assert_called_once_with("awesome-project:/some/dir") - - -def test_not_overwrite_nginx_config(): - """ - Tests wether the nginx config does not get - overwritten - """ - jinja_template_mock = "{{ project }}:{{ dir_path }}" - with patch("builtins.open", mock_open(read_data=jinja_template_mock)), patch.object(Path, "exists") as mock_exists, patch.object( - Path, "open", mock_open() - ) as mock_config, patch("subprocess.run"): - mock_exists.return_value = True - - create_nginx_config("awesome-project", "/some/dir") - - assert not mock_config.called - - def test_archive_artifact(): target_file = Path("/some/zipfile.zip") destination = "/tmp/null" @@ -101,23 +69,21 @@ def test_archive_artifact(): def test_remove_version(temp_project_version): - docs, config = temp_project_version("project", "1.0") - with patch("docat.utils.UPLOAD_FOLDER", docs), patch("docat.utils.NGINX_CONFIG_PATH", config): + docs = temp_project_version("project", "1.0") + with patch("docat.utils.UPLOAD_FOLDER", docs): remove_docs("project", "1.0") assert docs.exists() assert not (docs / "project").exists() - assert config.exists() - assert not (config / "project-doc.conf").exists() def test_remove_symlink_version(temp_project_version): project = "project" - docs, config = temp_project_version(project, "1.0") + docs = temp_project_version(project, "1.0") symlink_to_latest = docs / project / "latest" assert symlink_to_latest.is_symlink() - with patch("docat.utils.UPLOAD_FOLDER", docs), patch("docat.utils.NGINX_CONFIG_PATH", config): + with patch("docat.utils.UPLOAD_FOLDER", docs): remove_docs(project, "latest") assert not symlink_to_latest.exists() diff --git a/web/README.md b/web/README.md index 60fc3d26..d9ab5266 100644 --- a/web/README.md +++ b/web/README.md @@ -57,7 +57,6 @@ web frontend, you can mount the `dist/` folder as a docker volume: sudo docker run \ --detach \ --volume /path/to/doc:/var/docat/doc/ \ - --volume /path/to/locations:/etc/nginx/locations.d/ \ --volume /path/to/docat/web/dist:/var/www/html/ \ --publish 8000:80 \ docat diff --git a/web/src/components/ProjectOverview.vue b/web/src/components/ProjectOverview.vue index 2fc2f6c6..fbbbe96e 100644 --- a/web/src/components/ProjectOverview.vue +++ b/web/src/components/ProjectOverview.vue @@ -3,8 +3,8 @@ diff --git a/web/src/pages/Delete.vue b/web/src/pages/Delete.vue index e304ebed..23576821 100644 --- a/web/src/pages/Delete.vue +++ b/web/src/pages/Delete.vue @@ -82,7 +82,7 @@ export default { this.sending = true try { - await ProjectRepository.delete_doc(this.form.project, this.form.version, this.form.token) + await ProjectRepository.deleteDoc(this.form.project, this.form.version, this.form.token) const msg = "Documentation for " + this.form.project + " (" + this.form.version + ") deleted" this.clearForm() this.showError = true diff --git a/web/src/pages/Docs.vue b/web/src/pages/Docs.vue index b72b2344..9c0abb92 100644 --- a/web/src/pages/Docs.vue +++ b/web/src/pages/Docs.vue @@ -16,10 +16,10 @@ > - {{ version }} + {{ version.name + (version.tags.length ? ` (${version.tags.join(', ')})` : '') }} @@ -53,7 +53,7 @@ export default { document.title = this.$route.params.project + " | docat" this.versions = (await ProjectRepository.getVersions( this.$route.params.project - )).map((version) => version.name).sort(ProjectRepository.compareVersions) + )).sort((a, b) => ProjectRepository.compareVersions(a.name, b.name)) if (!this.selectedVersion) { this.selectedVersion = (this.versions.find((version) => version == 'latest') || this.versions[0]); @@ -74,7 +74,7 @@ export default { a.setAttribute('target', '_blank') } }) - + if(this.docURL) { this.load(docsFrame.contentWindow.location.href) } diff --git a/web/src/repositories/ProjectRepository.js b/web/src/repositories/ProjectRepository.js index 3d57b585..c10c2083 100644 --- a/web/src/repositories/ProjectRepository.js +++ b/web/src/repositories/ProjectRepository.js @@ -24,8 +24,8 @@ export default { * Returns all projects */ async get() { - const resp = await fetch(`${this.baseURL}/${resource}/`) - return await resp.json() + const resp = await fetch(`${this.baseURL}/api/projects`) + return (await resp.json()).projects }, /** @@ -69,9 +69,8 @@ export default { * @param {string} projectName Name of the project */ async getVersions(projectName) { - const resp = await fetch(`${this.baseURL}/${resource}/${projectName}/`) - return (await resp.json()) - .filter((version) => version.type == 'directory') + const resp = await fetch(`${this.baseURL}/api/projects/${projectName}`) + return (await resp.json()).versions }, /** @@ -112,7 +111,7 @@ export default { * @param {string} projectName Name of the project * @param {string} version Name of the version */ - async delete_doc(projectName, version, token) { + async deleteDoc(projectName, version, token) { const headers = { "Docat-Api-Key": token } const resp = await fetch(`${this.baseURL}/api/${projectName}/${version}`, { @@ -130,14 +129,16 @@ export default { /** * Compare two versions according to semantic version (semver library) * Will always consider the version latest as higher version - * + * * @param {string} versionNameA Name of the version one * @param {string} versionNameB Name of the version two */ compareVersions(versionNameA, versionNameB) { - if (versionNameA == "latest") return 1; - else if (versionNameB == "latest") return -1; - else { + if (versionNameA == "latest") { + return 1; + } else if (versionNameB == "latest") { + return -1; + } else { const versionA = semver.coerce(versionNameA); const versionB = semver.coerce(versionNameB); if (!versionA || !versionB) { @@ -145,5 +146,5 @@ export default { } return semver.compare(versionA, versionB); } - }, + } } diff --git a/web/tests/unit/project-repository.spec.js b/web/tests/unit/project-repository.spec.js index 28f78368..30373500 100644 --- a/web/tests/unit/project-repository.spec.js +++ b/web/tests/unit/project-repository.spec.js @@ -13,7 +13,7 @@ const mockFetchData = (fetchData) => { const mockFetchError = (error = "Error") => { global.fetch = jest.fn().mockImplementation(() => Promise.resolve({ ok: false, - json: () => Promise.resolve({message: error}) + json: () => Promise.resolve({ message: error }) })) } @@ -33,16 +33,18 @@ describe('ProjectRepository', () => { it('should get all projects', async () => { - const projects = [ - { name: 'awesome-project' }, - { name: 'pet-project' } - ] + const projects = { + 'projects': [ + 'awesome-project', + 'pet-project' + ] + } mockFetchData(projects) const result = await ProjectRepository.get() expect(global.fetch).toHaveBeenCalledTimes(1) - expect(result).toEqual(projects) + expect(result).toEqual(projects.projects) }) @@ -57,17 +59,19 @@ describe('ProjectRepository', () => { it('should get all versions of a project', async () => { - const versions = [ - { name: '1.0', type: 'directory' }, - { name: '2.0', type: 'directory' }, - { name: 'image.png', type: 'file' } - ] + const versions = { + 'name': 'awesome--project', + 'versions': [ + { name: '1.0', tags: [] }, + { name: '2.0', type: [] }, + ] + } mockFetchData(versions) const result = await ProjectRepository.getVersions('awesome--project') expect(global.fetch).toHaveBeenCalledTimes(1) - expect(result).toEqual(versions.filter((version) => version.type == 'directory')) + expect(result).toEqual(versions.versions) }) @@ -98,7 +102,7 @@ describe('ProjectRepository', () => { it('should delete an existing documentation', async () => { mockFetchData({}) - await ProjectRepository.delete_doc('awesome-project', '1.2', '1234') + await ProjectRepository.deleteDoc('awesome-project', '1.2', '1234') expect(global.fetch).toHaveBeenCalledTimes(1) expect(global.fetch).toHaveBeenCalledWith('https://do.cat/api/awesome-project/1.2', @@ -114,7 +118,7 @@ describe('ProjectRepository', () => { mockFetchError(errorMessage) - expect(ProjectRepository.delete_doc('existing-project', '4.0', { data: true })).rejects.toThrow(errorMessage) + expect(ProjectRepository.deleteDoc('existing-project', '4.0', { data: true })).rejects.toThrow(errorMessage) expect(global.fetch).toHaveBeenCalledTimes(1) }) diff --git a/web/vue.config.js b/web/vue.config.js index c704a2a4..c083c40e 100644 --- a/web/vue.config.js +++ b/web/vue.config.js @@ -5,6 +5,19 @@ module.exports = { devServer: { contentBase: path.join(__dirname, 'dist'), writeToDisk: true, + // proxy all requests to `/api/` and `/doc` for development. + // this fixes the CORS issue for iframe loading and other requests + // to the backend + proxy: { + '^/api': { + target: 'http://localhost:5000', + changeOrigin: true + }, + '^/doc': { + target: 'http://localhost:5000', + changeOrigin: true + } + } }, module: { rules: [{