From 8d487013224a9572b54ae9eced02c4d5bb9e8d3b Mon Sep 17 00:00:00 2001 From: Suraj Jacob <28795567+jacobmsft@users.noreply.github.com> Date: Tue, 7 Mar 2023 12:18:18 -0800 Subject: [PATCH 1/5] increment ubuntu container version move from pygithub to ghapi --- Dockerfile | 9 ++++----- container/get-latest-codeql-version.py | 8 ++------ container/libs/codeql.py | 8 ++++---- container/libs/github.py | 18 ++++++++++-------- container/requirements.txt | 3 ++- container/setup.py | 2 +- container/startup.py | 1 - 7 files changed, 23 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index d4d5742..6dc8516 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 AS codeql_base +FROM ubuntu:23.04 AS codeql_base LABEL maintainer="Github codeql team" # tzdata install needs to be non-interactive @@ -16,7 +16,7 @@ RUN apt-get update && \ build-essential \ unzip \ apt-transport-https \ - python3.8 \ + python3.10 \ python3-venv \ python3-pip \ python3-setuptools \ @@ -43,15 +43,14 @@ RUN cd /tmp && \ apt-get install -y default-jdk apt-transport-https && \ apt-get update && \ rm packages-microsoft-prod.deb -RUN apt-get install -y dotnet-sdk-3.1 +RUN apt-get install -y dotnet-sdk-7.0 # Clone our setup and run scripts #RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts RUN mkdir -p /usr/local/startup_scripts RUN ls -al /usr/local/startup_scripts COPY container /usr/local/startup_scripts/ -RUN pip3 install --upgrade pip \ - && pip3 install -r /usr/local/startup_scripts/requirements.txt +RUN pip3 install -r /usr/local/startup_scripts/requirements.txt # Install latest codeQL ENV CODEQL_HOME /usr/local/codeql-home diff --git a/container/get-latest-codeql-version.py b/container/get-latest-codeql-version.py index d2972bd..8229c05 100644 --- a/container/get-latest-codeql-version.py +++ b/container/get-latest-codeql-version.py @@ -1,13 +1,9 @@ #!/usr/bin/env python3 # get the parent directory of the script, to link libs - -import os -import sys - from libs.github import get_latest_github_repo_version def main(): - latest_release = get_latest_github_repo_version("github/codeql-cli-binaries") - print(latest_release.title) + latest_release = get_latest_github_repo_version("github", "codeql-cli-binaries") + print(latest_release.tag_name) main() diff --git a/container/libs/codeql.py b/container/libs/codeql.py index 7b6fba0..006c3bf 100644 --- a/container/libs/codeql.py +++ b/container/libs/codeql.py @@ -34,15 +34,15 @@ def download_and_install_latest_codeql(self, github_version): download_url = None download_path = None if os_name == 'posix': - download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-linux64.zip' + download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-linux64.zip' download_path = f'{self.TEMP_DIR}/codeql_linux.zip' elif os_name == 'nt': - download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-win64.zip' + download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-win64.zip' download_path = f'{self.TEMP_DIR}/codeql_windows.zip' else: exit(self.ERROR_UNKNOWN_OS) - logger.info(f'Downloading codeql-cli version {github_version.title}...') + logger.info(f'Downloading codeql-cli version {github_version.tag_name}...') check_output_wrapper(f"wget -q {download_url} -O {download_path}", shell=True).decode("utf-8") self.install_codeql_cli(download_path) #rm /tmp/codeql_linux.zip @@ -77,7 +77,7 @@ def get_current_local_version(self): return version def get_latest_codeql_github_version(self): - return get_latest_github_repo_version("github/codeql-cli-binaries") + return get_latest_github_repo_version("github", "codeql-cli-binaries") def install_codeql_cli(self, download_path): logger.info("Installing codeql-cli...") diff --git a/container/libs/github.py b/container/libs/github.py index 32547f9..6525870 100644 --- a/container/libs/github.py +++ b/container/libs/github.py @@ -1,18 +1,20 @@ from datetime import datetime, MINYEAR -from github import Github, GitRelease, Repository, GithubException +from ghapi.all import GhApi +from datetime import datetime, timezone +from dateutil import parser -def get_latest_github_repo_version(repo): - client = Github() - repo = client.get_repo(repo) - releases = repo.get_releases() +def get_latest_github_repo_version(owner, repository): + api = GhApi(owner=owner, repo=repository) + releases = api.repos.list_releases() latest_release = get_latest_github_release(releases) return latest_release def get_latest_github_release(releases): latest_release = None - latest_date = datetime(MINYEAR, 1, 1) + latest_date = datetime(MINYEAR, 1, 1).replace(tzinfo=timezone.utc) for release in releases: - if release.created_at > latest_date: - latest_date = release.created_at + release_date = parser.parse(release.created_at) + if release_date > latest_date: + latest_date = release_date latest_release = release return latest_release diff --git a/container/requirements.txt b/container/requirements.txt index f15ff22..ce3acfc 100644 --- a/container/requirements.txt +++ b/container/requirements.txt @@ -1 +1,2 @@ -PyGithub==1.43.7 \ No newline at end of file +ghapi==1.0.3 +python-dateutil==2.8.2 \ No newline at end of file diff --git a/container/setup.py b/container/setup.py index c3a0843..22a536e 100755 --- a/container/setup.py +++ b/container/setup.py @@ -44,7 +44,7 @@ def get_latest_codeql(args): current_installed_version = codeql.get_current_local_version() logger.info(f'Current codeql version: {current_installed_version}') latest_online_version = codeql.get_latest_codeql_github_version() - if current_installed_version != latest_online_version.title and args.check_latest_cli: + if current_installed_version != latest_online_version.tag_name and args.check_latest_cli: # we got a newer version online, download and install it codeql.download_and_install_latest_codeql(latest_online_version) # get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?) diff --git a/container/startup.py b/container/startup.py index 94319d7..dc05277 100755 --- a/container/startup.py +++ b/container/startup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os -import sys from time import sleep from libs.utils import get_env_variable, check_output_wrapper, get_logger from libs.codeql import * From ecf5df881163a83204ad021c35a2e7d4e473c9c0 Mon Sep 17 00:00:00 2001 From: Suraj Jacob <28795567+jacobmsft@users.noreply.github.com> Date: Tue, 7 Mar 2023 12:31:40 -0800 Subject: [PATCH 2/5] nit --- container/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/setup.py b/container/setup.py index 7fa9771..56a148f 100644 --- a/container/setup.py +++ b/container/setup.py @@ -46,7 +46,7 @@ def get_latest_codeql(args): # ensure we only query for the latest codeql cli version if we might actually update it if args.check_latest_cli: latest_online_version = codeql.get_latest_codeql_github_version() - if current_installed_version != latest_online_version.tag_name and args.check_latest_cli: + if current_installed_version != latest_online_version.tag_name: # we got a newer version online, download and install it codeql.download_and_install_latest_codeql(latest_online_version) # get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?) From 85a46506b81ebac4df07c24aaaad4f92cdfd219a Mon Sep 17 00:00:00 2001 From: Suraj Jacob <28795567+jacobmsft@users.noreply.github.com> Date: Tue, 7 Mar 2023 15:20:24 -0800 Subject: [PATCH 3/5] update with review comments --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 26bd71d..cd5b569 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:23.04 AS codeql_base +FROM ubuntu:22.04 AS codeql_base LABEL maintainer="Github codeql team" # tzdata install needs to be non-interactive @@ -47,7 +47,7 @@ RUN cd /tmp && \ apt-get install -y default-jdk apt-transport-https && \ apt-get update && \ rm packages-microsoft-prod.deb -RUN apt-get install -y dotnet-sdk-7.0 +RUN apt-get install -y dotnet-sdk-6.0 # Clone our setup and run scripts #RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts From d252b91a457c34b0d25feddc211cbc27285f1ee3 Mon Sep 17 00:00:00 2001 From: Suraj Jacob <28795567+jacobmsft@users.noreply.github.com> Date: Tue, 14 Mar 2023 10:14:59 -0700 Subject: [PATCH 4/5] remove duplicate cloning --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9d2eb5b..8dee718 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,6 @@ RUN cd /tmp && \ RUN apt-get install -y dotnet-sdk-6.0 # Clone our setup and run scripts -RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts RUN mkdir -p /usr/local/startup_scripts COPY container /usr/local/startup_scripts/ From a4d0b34dc0ab871fd505e798df64fe0bfc3e4360 Mon Sep 17 00:00:00 2001 From: Suraj Jacob <28795567+jacobmsft@users.noreply.github.com> Date: Tue, 14 Mar 2023 14:35:19 -0700 Subject: [PATCH 5/5] skip the chown step - not needed --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8dee718..3022669 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,8 +79,8 @@ RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-repo/*/ql/src/codeql- ENV PYTHONIOENCODING=utf-8 # Change ownership of all files and directories within CODEQL_HOME to the codeql user -RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME} +#RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME} USER ${USERNAME} -ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"] \ No newline at end of file +ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"] \ No newline at end of file