diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8ea22e9..8bf1b4c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ jobs: id: docker_build run: | echo ${{ secrets.GHCR_TOKEN }} | docker login ghcr.io -u GitHubSecurityLab --password-stdin - python release_tools/publish_docker.py release.txt main.py ${{ env.REGISTRY }}/${{ env.USER }}/${{ env.IMAGE_NAME }} ${{ github.event.inputs.release_tag }} + python release_tools/publish_docker.py ${{ env.REGISTRY }}/${{ env.USER }}/${{ env.IMAGE_NAME }} ${{ github.event.inputs.release_tag }} DIGEST=$(cat /tmp/digest.txt) echo "digest=$DIGEST" >> $GITHUB_OUTPUT diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..8ed924f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,48 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + curl \ + unzip \ + git \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install Docker CLI (debian) +RUN apt-get update \ + && install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc \ + && chmod a+r /etc/apt/keyrings/docker.asc \ + && echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null \ + && apt-get update && apt-get install -y docker-ce-cli \ + && rm -rf /var/lib/apt/lists/* + +# Install GitHub CLI +RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ + && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt-get update \ + && apt-get install -y gh \ + && rm -rf /var/lib/apt/lists/* + +# Install CodeQL CLI +RUN curl -Ls -o /tmp/codeql.zip https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql-linux64.zip \ + && unzip /tmp/codeql.zip -d /opt \ + && mv /opt/codeql /opt/codeql-cli \ + && ln -s /opt/codeql-cli/codeql /usr/local/bin/codeql \ + && rm /tmp/codeql.zip + +# Install seclab-taskflow-agent from PyPI +RUN pip install seclab-taskflow-agent + +# Install CodeQL pack dependencies +RUN export SECLAB_TASKFLOW_AGENT=$(python -c 'import seclab_taskflow_agent as x; print(x.__path__[0])') && \ + codeql pack install $SECLAB_TASKFLOW_AGENT/mcp_servers/codeql/queries/mcp-cpp && \ + codeql pack install $SECLAB_TASKFLOW_AGENT/mcp_servers/codeql/queries/mcp-js + +ENTRYPOINT ["python", "-m", "seclab_taskflow_agent"] diff --git a/release.txt b/release.txt deleted file mode 100644 index 0679154..0000000 --- a/release.txt +++ /dev/null @@ -1,122 +0,0 @@ -# .ignore files -./.gitignore -./.dockerignore - -# release tooling -./release.txt -./release_tools/release.sh -./release_tools/HOWTO.md -./release_tools/copy_files.py -./release_tools/publish_docker.py - -# license, etc. -./LICENSE -./CODEOWNERS -./CONTRIBUTING.md -./NOTICE -./CODE_OF_CONDUCT.md -./SUPPORT.md -./SECURITY.md - -# deps -./pyproject.toml - -# docker -./docker/run.sh - -# docs -./README.md -./doc/GRAMMAR.md - -# logs -./logs/ABOUT - -# code -./src/seclab_taskflow_agent/__init__.py -./src/seclab_taskflow_agent/__about__.py -./src/seclab_taskflow_agent/__main__.py -./src/seclab_taskflow_agent/agent.py -./src/seclab_taskflow_agent/capi.py -./src/seclab_taskflow_agent/mcp_utils.py -./src/seclab_taskflow_agent/env_utils.py -./src/seclab_taskflow_agent/render_utils.py -./src/seclab_taskflow_agent/shell_utils.py -./src/seclab_taskflow_agent/available_tools.py - -# configs -./examples/model_configs/model_config.yaml - -# prompts -./examples/prompts/example_prompt.yaml - -# taskflows -./examples/taskflows/CVE-2023-2283.yaml -./examples/taskflows/example_reusable_prompt.yaml -./examples/taskflows/echo.yaml -./examples/taskflows/example.yaml -./examples/taskflows/example_inputs.yaml -./examples/taskflows/example_repeat_prompt.yaml -./examples/taskflows/example_repeat_prompt_dictionary.yaml -./examples/taskflows/example_reusable_taskflows.yaml -./examples/taskflows/example_large_list_result_iter.yaml -./examples/taskflows/example_repeat_prompt_async.yaml -./examples/taskflows/single_step_taskflow.yaml -./examples/taskflows/example_triage_taskflow.yaml -./examples/taskflows/example_globals.yaml - -# toolboxes -## github official -./src/seclab_taskflow_agent/toolboxes/github_official.yaml -## echo -./src/seclab_taskflow_agent/toolboxes/echo.yaml -./src/seclab_taskflow_agent/mcp_servers/echo/echo.py -## memcache -./src/seclab_taskflow_agent/toolboxes/memcache.yaml -./src/seclab_taskflow_agent/mcp_servers/memcache/__init__.py -./src/seclab_taskflow_agent/mcp_servers/memcache/memcache.py -./src/seclab_taskflow_agent/mcp_servers/memcache/memcache_backend/__init__.py -./src/seclab_taskflow_agent/mcp_servers/memcache/memcache_backend/backend.py -./src/seclab_taskflow_agent/mcp_servers/memcache/memcache_backend/dictionary_file.py -./src/seclab_taskflow_agent/mcp_servers/memcache/memcache_backend/sql_models.py -./src/seclab_taskflow_agent/mcp_servers/memcache/memcache_backend/sqlite.py -## logbook -./src/seclab_taskflow_agent/toolboxes/logbook.yaml -./src/seclab_taskflow_agent/mcp_servers/logbook/logbook.py -## codeql -./src/seclab_taskflow_agent/toolboxes/codeql.yaml -./src/seclab_taskflow_agent/mcp_servers/codeql/mcp_server.py -./src/seclab_taskflow_agent/mcp_servers/codeql/client.py -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/README.md -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-js/relative_to_absolute.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-js/qlpack.yml -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-js/absolute_to_relative.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-js/definition_location_for_function.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-js/call_graph_from.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-js/locations.qll -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-js/call_graph_to.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/stmt_location.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/declaration_location_for_variable.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/relative_to_absolute.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/call_graph_from_to.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/qlpack.yml -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/absolute_to_relative.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/codeql-pack.lock.yml -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/definition_location_for_function.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/call_graph_from.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/list_functions.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/locations.qll -./src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp/call_graph_to.ql -./src/seclab_taskflow_agent/mcp_servers/codeql/jsonrpyc/LICENSE -./src/seclab_taskflow_agent/mcp_servers/codeql/jsonrpyc/__meta__.py -./src/seclab_taskflow_agent/mcp_servers/codeql/jsonrpyc/__init__.py -./src/seclab_taskflow_agent/mcp_servers/codeql/jsonrpyc/py.typed - -# personalities -./src/seclab_taskflow_agent/personalities/assistant.yaml -./src/seclab_taskflow_agent/personalities/c_auditer.yaml -./examples/personalities/fruit_expert.yaml -./examples/personalities/echo.yaml -./examples/personalities/banana_expert.yaml -./examples/personalities/orange_expert.yaml -./examples/personalities/example_triage_agent.yaml -./examples/personalities/apple_expert.yaml diff --git a/release_tools/HOWTO.md b/release_tools/HOWTO.md index 2dae984..d15c943 100644 --- a/release_tools/HOWTO.md +++ b/release_tools/HOWTO.md @@ -2,13 +2,11 @@ To release an updated version of the Agent perform the following steps: -1. Add any newly created files or dependencies to `release.txt`. - -2. Release an updated Docker image: +1. Release an updated Docker image: ```sh docker login ghcr.io -u YOUR_GITHUB_USERNAME -python release_tools/publish_docker.py release.txt main.py ghcr.io/githubsecuritylab/seclab-taskflow-agent latest +python release_tools/publish_docker.py ghcr.io/githubsecuritylab/seclab-taskflow-agent latest ``` Note: your login password is a GitHub PAT with packages write/read/delete scope enabled. diff --git a/release_tools/publish_docker.py b/release_tools/publish_docker.py index ecaebb9..5644eea 100644 --- a/release_tools/publish_docker.py +++ b/release_tools/publish_docker.py @@ -5,85 +5,6 @@ import shutil import subprocess import sys -import tempfile - -def read_file_list(list_path): - """ - Reads a file containing file paths, ignoring empty lines and lines starting with '#'. - Returns a list of relative file paths. - """ - with open(list_path, "r") as f: - lines = [line.strip() for line in f] - return [line for line in lines if line and not line.startswith("#")] - -def copy_files_to_dir(file_list, dest_dir): - """ - Copies files to dest_dir, preserving their relative paths. - """ - for rel_path in file_list: - abs_src = os.path.abspath(rel_path) - abs_dest = os.path.abspath(os.path.join(dest_dir, rel_path)) - os.makedirs(os.path.dirname(abs_dest), exist_ok=True) - shutil.copy2(abs_src, abs_dest) - -def write_dockerfile(dest_dir, entrypoint): - """ - Writes a Dockerfile that installs Python dependencies, GitHub CLI, and CodeQL CLI. - """ - dockerfile = f''' -FROM python:3.11-slim - -WORKDIR /app - -# Install system dependencies -RUN apt-get update && apt-get install -y \\ - curl \\ - unzip \\ - git \\ - ca-certificates \\ - && rm -rf /var/lib/apt/lists/* - -# Install Docker CLI (debian) -RUN apt-get update \\ - && install -m 0755 -d /etc/apt/keyrings \\ - && curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc \\ - && chmod a+r /etc/apt/keyrings/docker.asc \\ - && echo \\ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \\ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \\ - tee /etc/apt/sources.list.d/docker.list > /dev/null \\ - && apt-get update && apt-get install -y docker-ce-cli \\ - && rm -rf /var/lib/apt/lists/* - -# Install GitHub CLI -RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \\ - && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \\ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \\ - && apt-get update \\ - && apt-get install -y gh \\ - && rm -rf /var/lib/apt/lists/* - -# Install CodeQL CLI -RUN curl -Ls -o /tmp/codeql.zip https://github.com/github/codeql-cli-binaries/releases/latest/download/codeql-linux64.zip \\ - && unzip /tmp/codeql.zip -d /opt \\ - && mv /opt/codeql /opt/codeql-cli \\ - && ln -s /opt/codeql-cli/codeql /usr/local/bin/codeql \\ - && rm /tmp/codeql.zip - -COPY . /app - -# Install CodeQL pack dependencies -RUN codeql pack install /app/src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-cpp -RUN codeql pack install /app/src/seclab_taskflow_agent/mcp_servers/codeql/queries/mcp-js - -# Install Python dependencies if pyproject.toml exists -RUN pip install hatch -RUN if [ -f pyproject.toml ]; then hatch run sync-deps; fi - -ENTRYPOINT ["hatch", "run", "{entrypoint}"] -''' - with open(os.path.join(dest_dir, "Dockerfile"), "w") as f: - f.write(dockerfile) def get_image_digest(image_name, tag): result = subprocess.run( @@ -111,23 +32,13 @@ def build_and_push_image(dest_dir, image_name, tag): f.write(digest) if __name__ == "__main__": - if len(sys.argv) != 5: - print("Usage: python build_and_publish_docker.py ") - print("Example: python build_and_publish_docker.py files.txt main.py ghcr.io/anticomputer/my-python-app latest") + if len(sys.argv) != 3: + print("Usage: python build_and_publish_docker.py ") + print("Example: python build_and_publish_docker.py ghcr.io/anticomputer/my-python-app latest") sys.exit(1) - file_list_path = sys.argv[1] - entrypoint_py = sys.argv[2] - image_name = sys.argv[3] - tag = sys.argv[4] - - # Read file paths - file_list = read_file_list(file_list_path) + image_name = sys.argv[1] + tag = sys.argv[2] - with tempfile.TemporaryDirectory() as build_dir: - # Copy files - copy_files_to_dir(file_list, build_dir) - # Write Dockerfile - write_dockerfile(build_dir, entrypoint_py) - # Build and push image - build_and_push_image(build_dir, image_name, tag) + # Build and push image + build_and_push_image("docker", image_name, tag) diff --git a/release_tools/release.sh b/release_tools/release.sh index d80be5b..54d40d4 100755 --- a/release_tools/release.sh +++ b/release_tools/release.sh @@ -1,2 +1,2 @@ #!/bin/sh -python release_tools/publish_docker.py release.txt main ghcr.io/githubsecuritylab/seclab-taskflow-agent latest +python release_tools/publish_docker.py ghcr.io/githubsecuritylab/seclab-taskflow-agent latest