diff --git a/.gitignore b/.gitignore index f4df6301..6db38818 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ package/ .local local .DS_Store +*.whl + # Unit test __pytest_reports diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8111c485..00000000 --- a/Dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -#### -# This Dockerfile is used to build the container for charon -# -# charon requires python 3 -# -# 0. Step into the project dir -# -# 1. Build the image -# docker/podman build -t charon:1.0.0 . -# -# 2. Run the container as daemon, mount the host ~/upload/ path to container /root/upload/ path, -# the uploading path is the dir location where you will upload the tarballs from -# add -e to set specific environment variables, such as: AWS_PROFILE, aws_endpoint_url, bucket -# docker/podman run -dit -v ~/upload/:/root/upload/ --name charon charon:1.0.0 -# -# 3. Execute the container -# docker/podman exec -it charon bash -# -# 4. Start using uploader -# charon upload/delete from /root/upload/... -### - -# parser directive, always points to the latest release of the version 1 syntax, -# automatically checks for updates before building, making sure using the most current version -# syntax=docker/dockerfile:1 -FROM python:3.8 - -# ensure the latest version of pip -RUN pip3 install --no-cache-dir --upgrade pip - -RUN adduser charon -USER charon -WORKDIR /home/charon - -# pip respects TMPDIR to set another enough disk space for pip packages installation -ENV TMPDIR="/home/charon/tmp" - -# install all required packages -COPY --chown=charon:charon requirements.txt ./ -RUN pip3 install --user --no-cache-dir -r requirements.txt - -# prepare configs for charon -ADD ./config/charon.conf /home/charon/.charon/charon.conf -ADD ./config/aws-credentials /home/charon/.aws/credentials - -# prepare templates for charon -ADD ./template/index.html.j2 /home/charon/.charon/template/index.html.j2 -ADD ./template/maven-metadata.xml.j2 /home/charon/.charon/template/maven-metadata.xml.j2 - -ENV PATH="/home/charon/.local/bin:${PATH}" -COPY --chown=charon:charon . . - -# install charon -RUN pip3 install --user --no-cache-dir . - -# this will be invoked when container runs, charon will directly setup -# from the container and keep running as long as the bash is active -CMD ["bash"] diff --git a/README.md b/README.md index af0f6c89..d52a4506 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ This command will upload the repo in tarball to S3. It will auto-detect if the tarball is for maven or npm * For maven type, it will: + * Scan the tarball for all paths and collect them all. * Check the existence in S3 for all those paths. * Filter out the paths in tarball based on: @@ -70,10 +71,8 @@ It will auto-detect if the tarball is for maven or npm * Upload these artifacts to S3 with metadata of the product. * If the artifacts already exists in S3, update the metadata of the product by appending the new product. - * NPM type (TBH): We need to know the exact tarball structure of npm repo - * For both types, after uploading the files, regenerate/refresh the index files for these paths. @@ -107,10 +106,30 @@ This command will refresh the index.html for the specified path. * Note that if the path is a NPM metadata path which contains package.json, this refreshment will not work because this type of folder will display the package.json instead of the index.html in http request. -### charon-validate: validate the checksum of files in specified path in a maven repository +### charon-cf-check: check the invalidation status of the specified invalidation id for AWS CloudFront + +```bash +usage: charon cf check $invalidation_id [-t, --target] [-D, --debug] [-q, --quiet] +``` + +### charon-cf-invalidate: do invalidating on AWS CloudFront for the specified paths + +```bash +usage: charon cf invalidate [-t, --target] [-p, --path] [-f, --path-file] [-D, --debug] [-q, --quiet] +``` + +### charon-checksum-validate: validate the checksum of files in specified path in a maven repository ```bash -usage: charon validate $path [-t, --target] [-f, --report_file_path] [-i, --includes] [-r, --recursive] [-D, --debug] [-q, --quiet] +usage: charon checksum validate $path [-t, --target] [-f, --report_file_path] [-i, --includes] [-r, --recursive] [-D, --debug] [-q, --quiet] ``` This command will validate the checksum of the specified path for the maven repository. It will calculate the sha1 checksum of all artifact files in the specified path and compare with the companied .sha1 files of the artifacts, then record all mismatched artifacts in the report file. If some artifact files misses the companied .sha1 files, they will also be recorded. + +### charon-checksum-refresh: refresh the checksum files for the artifacts in the specified maven repository + +```bash +usage: charon checksum refresh [-t, --target] [-p, --path] [-f, --path-file] [-D, --debug] [-q, --quiet] +``` + +This command will refresh the checksum files for the specified artifact files in the maven repository. Sometimes the checksum files are not matched with the artifacts by some reason, so this command will do the refresh to make it match again. It will calculate the checksums of all artifact files in the specified path and compare with the companied checksum files of the artifacts, if the checksum are not matched, they will be refreshed. diff --git a/config/charon.yaml.sample b/config/charon.yaml.sample index 97c37d4a..b3e0fb71 100644 --- a/config/charon.yaml.sample +++ b/config/charon.yaml.sample @@ -1,3 +1,6 @@ +#aws_profile: ${profile} +#aws_cf_enable: True + ignore_patterns: - ".*^(redhat).*" - ".*snapshot.*" @@ -30,3 +33,5 @@ targets: - bucket: "stage-npm-npmjs" prefix: / registry: "npm.stage.registry.redhat.com" + +#manifest_bucket: manifest \ No newline at end of file diff --git a/image/Containerfile b/image/Containerfile new file mode 100644 index 00000000..6ee0c02c --- /dev/null +++ b/image/Containerfile @@ -0,0 +1,59 @@ +#### +# This Containerfile is used to build the container for charon +# +# charon requires python 3 +# +# 0. Step into the project dir +# +# 1. Build the image +# docker/podman build -t charon:1.0.0 -f image/Containerfile . +# +# 2. Run the container as daemon, mount the host ~/upload/ path to container /root/upload/ path, +# the uploading path is the dir location where you will upload the tarballs from +# add -e to set specific environment variables, such as: AWS_PROFILE, aws_endpoint_url, bucket +# docker/podman run -dit -v ~/upload/:/home/charon/upload/ --name charon charon:1.0.0 +# +# 3. Execute the container +# docker/podman exec -it charon bash +# +# 4. Start using uploader +# charon upload/delete from /home/charon/upload/... +### +FROM registry.access.redhat.com/ubi8-minimal:latest as builder + +ARG GIT_BRANCH=main + +RUN microdnf install -y git-core python3.12 python3.12-pip && microdnf clean all +RUN git clone -b ${GIT_BRANCH} --depth 1 https://github.com/Commonjava/charon.git +RUN pip3 install --no-cache-dir --upgrade pip +RUN pip3 wheel ./charon + +FROM registry.access.redhat.com/ubi8-minimal:latest + +ARG USER=charon +ARG UID=10000 +ARG HOME_DIR=/home/${USER} + +WORKDIR ${HOME_DIR} + +USER root + +RUN microdnf install -y python3.12 python3.12-pip shadow-utils && microdnf clean all +RUN useradd -d ${HOME_DIR} -u ${UID} -g 0 -m -s /bin/bash ${USER} \ + && chown ${USER}:0 ${HOME_DIR} \ + && chmod -R g+rwx ${HOME_DIR} \ + && chmod g+rw /etc/passwd + +COPY --from=builder ./*.whl ./ +RUN pip3 install --no-cache-dir --upgrade pip +RUN pip3 install --no-cache-dir ./*.whl +RUN rm ./*.whl + +RUN microdnf remove python3.12-pip shadow-utils && microdnf clean all + +USER ${USER} + +ENV HOME=${HOME_DIR} \ + LANG=en_US.UTF-8 + +CMD ["/usr/local/bin/charon"] diff --git a/requirements.txt b/requirements.txt index 4f1b20ad..a2f622f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,12 @@ -setuptools-rust==1.7.0 -Jinja2==3.1.4 -boto3==1.28.46 -botocore==1.31.46 -click==8.1.7 -requests==2.31.0 -PyYAML==6.0.1 -defusedxml==0.7.1 -subresource-integrity==0.2 -jsonschema==4.19.0 -urllib3==1.26.18 \ No newline at end of file +Jinja2>=3.1.4 +boto3>=1.28.46 +botocore>=1.31.46 +click>=8.1.7 +requests>=2.31.0 +PyYAML>=6.0.1 +defusedxml>=0.7.1 +subresource-integrity>=0.2 +jsonschema>=4.19.0 +urllib3>=1.26.18 +semantic-version>=2.10.0 +setuptools>=70.0.0 diff --git a/setup.py b/setup.py index 0c11c688..c09e3784 100755 --- a/setup.py +++ b/setup.py @@ -15,12 +15,8 @@ """ from setuptools import setup, find_packages -version = "1.3.1" +version = "1.3.2" -# f = open('README.md') -# long_description = f.read().strip() -# long_description = long_description.split('split here', 1)[1] -# f.close() long_description = """ This charon is a tool to synchronize several types of artifacts repository data to RedHat Ronda service (maven.repository.redhat.com). @@ -50,4 +46,18 @@ entry_points={ "console_scripts": ["charon = charon.cmd:cli"], }, + install_requires=[ + "Jinja2>=3.1.4", + "boto3>=1.28.46", + "botocore>=1.31.46", + "click>=8.1.7", + "requests>=2.31.0", + "PyYAML>=6.0.1", + "defusedxml>=0.7.1", + "subresource-integrity>=0.2", + "jsonschema>=4.19.0", + "urllib3>=1.26.18", + "semantic-version>=2.10.0", + "setuptools>=70.0.0", + ], ) diff --git a/tests/requirements.txt b/tests/requirements.txt index 4acad34d..43ac571a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -5,5 +5,5 @@ pytest-cov pytest-html flake8 requests-mock -moto==5.0.3 -python-gnupg==0.5.0 +moto>=5.0.3,<6 +python-gnupg>=0.5.0,<1