diff --git a/docs/docs/users/configuration/_category_.json b/docs/docs/users/configuration/_category_.json new file mode 100644 index 0000000..ba3ec97 --- /dev/null +++ b/docs/docs/users/configuration/_category_.json @@ -0,0 +1,9 @@ +{ + "label": "Configuration", + "position": 3, + "link": { + "type": "generated-index", + "description": "Configure leviathan" + }, + "collapsed": false +} diff --git a/docs/docs/users/configuration/config_yaml.md b/docs/docs/users/configuration/config_yaml.md new file mode 100644 index 0000000..f969045 --- /dev/null +++ b/docs/docs/users/configuration/config_yaml.md @@ -0,0 +1,82 @@ +--- +sidebar_position: 2 +title: config.yaml +--- + +# Configuration file + +Leviathan uses a `config.yaml` file for configration, this file is automatically created on first run with +some default values, everything about leviathan can be configured here ([except logging](#logging)) + +You can also modify some of the options using environment variables. + +Below is the default configuration file with explanations for each option. + +```yaml +clients: + # Enable local Docker execution for jobs + enable_local_docker: true + # SSH client configurations for remote execution + ssh: + example: + enable: false # Enable/disable this SSH client + host: 192.168.1.69 # Remote host address + port: 22 # SSH port + user: test # SSH username + password: "" # SSH password (if not using key auth) + remotepublickey: "" # Remote public key path + usepublickeyauth: false # Whether to use public key authentication +db: + postgres: + enable_postgres: false # Enable PostgreSQL instead of SQLite + postgres_db: leviathan # Database name + postgres_host: localhost # PostgreSQL host + postgres_pass: postgres # Database password + postgres_port: "5432" # Database port + postgres_ssl: disable # SSL mode (disable, require, verify-ca, verify-full) + postgres_user: postgres # Database user + sqlite: + db_path: ./appdata/config/leviathan.db # SQLite database file path +folder: + # All folder paths should remain within ./appdata to ensure proper hardlinking + job_output_dir: ./appdata/output # Directory for job outputs + labs: ./appdata/labs # Directory for lab files + log_dir: ./appdata/config/logs/leviathan.log # Log file location + ssh_config: ./appdata/config/ssh_config # SSH configuration directory + tmp_submission_dir: ./appdata/submissions # Temporary submission directory + tmp_uploads: ./appdata/tmp_uploads # Temporary upload directory +jobs: + concurrent_jobs: 50 # Maximum number of concurrent jobs +server: + log_level: info # Logging level (debug, info, warn, error) + port: "9221" # Server port + apikey: "" # API key for authentication +``` + +# Environment Variables + +Some configuration options can be overridden using environment variables. Here are the available environment variables: + +## Database Configuration + +Read more [here](database.md) + +## Directory Configuration + +Read more [here](folders.md) + +## Logging + +You can set the following logger options these are available using only env vars + +| Environment Variable | Description | Default Value | +|----------------------------------|----------------------------------------------------|---------------| +| `LEVIATHAN_LOG_LEVEL` | Set log level (debug, info, warn, error, disabled) | `info` | +| `LEVIATHAN_LOG_SHOW_CALLER_FILE` | Shows the file where the log was generated from | `false` | + +The logging system has the following behaviour: + +- Log rotation: Maximum file size of 10MB +- Backup retention: Keeps up to 5 backup files +- Age limit: Logs are kept for up to 30 days +- Compression: Old log files are automatically compressed diff --git a/docs/docs/users/configuration/database.md b/docs/docs/users/configuration/database.md new file mode 100644 index 0000000..cc1db98 --- /dev/null +++ b/docs/docs/users/configuration/database.md @@ -0,0 +1,28 @@ +--- +sidebar_position: 4 +title: Database +--- + +# Database options + +Leviathan uses sqlite by default this makes development and testing easier, but it also supports postgres. + +## PostgresSQL + +To enable either modify the config.yaml or pass in the following environment variables + +| Environment Variable | Description | Default Value | +|----------------------|--------------------------|---------------| +| `POSTGRES_ENABLE` | Enable PostgreSQL | `false` | +| `POSTGRES_HOST` | PostgreSQL host | `"localhost"` | +| `POSTGRES_PORT` | PostgreSQL port | `"5432"` | +| `POSTGRES_USER` | PostgreSQL user | `"postgres"` | +| `POSTGRES_PASSWORD` | PostgreSQL password | `"postgres"` | +| `POSTGRES_DB` | PostgreSQL database name | `"postgres"` | +| `POSTGRES_SSL` | PostgreSQL SSL mode | `"disable"` | + +## Sqlite + +No additional setup is required for sqlite. + +If you wish to keep using sqlite, set `postgres_enable: false` and leave the postgres option as is. \ No newline at end of file diff --git a/docs/docs/users/configuration/docker_hosts.md b/docs/docs/users/configuration/docker_hosts.md new file mode 100644 index 0000000..5db2ce4 --- /dev/null +++ b/docs/docs/users/configuration/docker_hosts.md @@ -0,0 +1,96 @@ +--- +sidebar_position: 3 +title: Docker Hosts +--- + +# Connecting to Docker Hosts + +Leviathan uses SSH to connect to remote docker hosts and supports multiple ways to connect to Docker hosts. + +This can be configured in the `clients` section in `config.yaml` +Here is an example ssh section + +```yaml +clients: + enable_local_docker: true + ssh: + example: # key is a nickname for this machine + enable: false + host: 192.168.1.69 + port: 22 + user: test + password: "" + remotepublickey: "" + usepublickeyauth: false +``` + +## SSH user considerations + +You must ensure that the ssh user that the leviathan uses can access the docker demon properly, + +You can create a user with limited permissions that can only control the Docker daemon remotely. This approach follows +the principle of the least privilege, providing only the specific access needed. + +## Connection hierarchy + +When leviathan connects to a docker host it follows the following order + +1. Checks if `use_public_key_auth` is enabled then uses [public key auth](#ssh-public-key-authentication-recommended) +2. If that is disabled, it checks the `passoword` section and uses [password auth](#ssh-password-authentication) +3. If password is empty, then it will fallback to [host public key auth](#ssh-public-key-authentication-host-dependent) + +Regardless of the auth method, if the host public key is automatically stored on first +connection and verified on subsequent connections. + +If the `remotepublickey` is set before first connect then it will be used to verify. + +**Leviathan requires at least 1 docker host connected at anytime, +if it is unable to connect to any docker host then it will exit with a fatal error.** + +## Authentication Methods + +### SSH Public Key Authentication (Recommended) + +```yaml +use_public_key_auth: true +``` + +- Uses auto-generated key pair stored in Leviathan's config directory, found in `appdata/config/ssh_config` +- Requires adding Leviathan's public key to remote host's `~/.ssh/authorized_keys` +- No password required + +### SSH Password Authentication + +```yaml +password: "some password" +``` + +- Basic authentication method +- Recommended for initial setup/testing only + +### SSH Public Key Authentication (Host dependent) + +- Uses key pair stored on the machine running leviathan typically at `~/.ssh` +- Requires mounting the `.ssh` dir into the docker container or +- transferring the keys manually and setting up `.ssh` inside the container file system + +### Local Docker Connection + +- Enabled by default on first run. +- Requires Docker daemon running on local machine +- Requires mounting `/var/run/docker.sock:/var/run/docker.sock` into the container +- Disable with `enable_local_docker: false` in config.yaml + +# Configuration Parameters + +| Parameter | Required | Default | Description | Notes | +|-----------------------|:--------:|:-------:|----------------------------------|----------------------------------------------------| +| `enable` | No | `true` | Host availability | Set to `false` to disable host | +| `host` | Yes | - | Host machine IP | `http://192.168.1.1` or `https:remote.docker.com` | +| `port` | No | `22` | SSH port number | Standard SSH port used if not specified | +| `user` | Yes | - | SSH login username | | +| `password` | No | - | SSH password | Used for password-based authentication | +| `remote_public_key` | No | - | Remote host's public key | Automatically set by Leviathan on first connection | +| `use_public_key_auth` | No | `false` | Enable public key authentication | Set to `true` to use leviathan's SSH key pairs | + + diff --git a/docs/docs/users/configuration/folders.md b/docs/docs/users/configuration/folders.md new file mode 100644 index 0000000..6adf151 --- /dev/null +++ b/docs/docs/users/configuration/folders.md @@ -0,0 +1,43 @@ +--- +sidebar_position: 5 +title: Folders +--- + +# Folder config + +Leviathan uses the following folders for its operations. + +| Environment Variable | Description | Purpose/Usage | Default Value | +|-------------------------|------------------------------|--------------------------------------------------------------------------------|-------------------------------| +| `TMP_SUBMISSION_DIR` | Temporary submission storage | Stores job submissions temporarily; contents are deleted after job completion | `./appdata/submissions` | +| `SUBMISSION_OUTPUT_DIR` | Job output storage | Preserves stdout and logs from containers where jobs executed | `./appdata/output` | +| `SSH_CONFIG_DIR` | SSH configuration storage | Contains Leviathan-managed SSH keys for remote Docker host connections | `./appdata/config/ssh_config` | +| `LABS_DIR` | Laboratory files storage | Stores lab files | `./appdata/labs` | +| `TMP_UPLOAD_DIR` | Temporary upload storage | Facilitates file transfers to Leviathan; contents are deleted after processing | `./appdata/tmp_uploads` | + +# Folder Configuration + +Leviathan uses a structured directory system for its operations. + +## Hardlinks + +Leviathan uses hardlinks for efficient job processing. Changing folder locations requires special consideration: + +- 🚫 **Avoid network mounts** - NFS/SMB volumes often lack proper hardlink support +- 📂 **Maintain same filesystem** - All paths must reside on the same physical storage device + +### Docker Deployment Considerations + +When deploying via Docker, volume mounting requires special attention. For example: + +`/some_host_mount:/appdata/folder1` + +`/some_host_mount:/appdata/folder2` + +Docker treats these paths as separate filesystems, preventing hardlinks between them. + +❗ Changing these paths incorrectly may lead to: + +- Failed job submissions +- Corrupted submission artifacts +- Incomplete output generation diff --git a/docs/docs/users/configuration/intro.md b/docs/docs/users/configuration/intro.md new file mode 100644 index 0000000..8e1fecc --- /dev/null +++ b/docs/docs/users/configuration/intro.md @@ -0,0 +1,26 @@ +--- +sidebar_position: 1 +title: Intro +--- + +# Leviathan Configuration + +Leviathan creates an `appdata` directory in its working directory upon startup. This directory stores all configuration and runtime data. + +For Docker deployments, the `appdata` directory is located at `/app/appdata` inside the container. To persist data between container restarts, you should mount this directory as a volume. + +## Docker Compose Example + +Below is a recommended `docker-compose.yml` configuration: + +```yaml +services: + leviathan: + container_name: leviathan + image: ghcr.io/makeopensource/leviathan:master + volumes: + - ./appdata:/app/appdata + ports: + - "9221:9221" + restart: unless-stopped +``` diff --git a/docs/docs/users/how-it-works.md b/docs/docs/users/how-it-works.md new file mode 100644 index 0000000..d819912 --- /dev/null +++ b/docs/docs/users/how-it-works.md @@ -0,0 +1,4 @@ +--- +sidebar_position: 2 +title: How it works +--- \ No newline at end of file diff --git a/docs/docs/users/installation/_category_.json b/docs/docs/users/installation/_category_.json index fe19551..286a831 100644 --- a/docs/docs/users/installation/_category_.json +++ b/docs/docs/users/installation/_category_.json @@ -3,7 +3,7 @@ "position": 2, "link": { "type": "generated-index", - "description": "Guide to installing leviathan on various platforms." + "description": "Guide to installing leviathan on various platforms." }, "collapsed": false } diff --git a/docs/docs/users/intro.md b/docs/docs/users/intro.md index 9f7aa58..7d32608 100644 --- a/docs/docs/users/intro.md +++ b/docs/docs/users/intro.md @@ -6,7 +6,20 @@ title: About # Leviathan -Leviathan is a container orchestrator and job runner intended to be code runner component of the [DevU](https://github.com/makeopensource/devU). +A container orchestrator and job runner for executing code in docker containers. + +## Overview + +Leviathan is the code execution engine for the [DevU project](https://github.com/makeopensource/devU). +It securely runs submitted code in isolated containers, providing evaluation results and feedback. +Designed as a modern replacement for [Autolab Tango](https://github.com/autolab/Tango). + +Leviathan maintains full compatibility with existing Autolab/Tango graders. Any graders written for Tango will run on Leviathan without modification. + +## Features +- Secure container-based execution +- Resource limiting and management +- gRPC API for type-safe, efficient communication ## How it Works diff --git a/docs/package.json b/docs/package.json index 6008244..76f02dc 100644 --- a/docs/package.json +++ b/docs/package.json @@ -12,6 +12,7 @@ "serve": "docusaurus serve", "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids", + "md": "echo '# New Markdown File' >", "typecheck": "tsc" }, "dependencies": {