From 87d8389dcbbb9678a26c3b9636ac760e2869c256 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 18 Apr 2024 15:36:55 -0700 Subject: [PATCH 1/3] Update docker compose files * Remove docker-compose_purldb_public.yml * Use docker-compose_traefik.yml as base for new docker-compose.yml Signed-off-by: Jono Yang --- ...se_purldb.yml => docker-compose.common.yml | 19 --- ...deio.yml => docker-compose.matchcodeio.yml | 0 docker-compose.yml | 35 ++++++ docker-compose_purldb_public.yml | 111 ------------------ docker-compose_traefik.yml | 18 --- 5 files changed, 35 insertions(+), 148 deletions(-) rename docker-compose_purldb.yml => docker-compose.common.yml (87%) rename docker-compose_matchcodeio.yml => docker-compose.matchcodeio.yml (100%) create mode 100644 docker-compose.yml delete mode 100644 docker-compose_purldb_public.yml delete mode 100644 docker-compose_traefik.yml diff --git a/docker-compose_purldb.yml b/docker-compose.common.yml similarity index 87% rename from docker-compose_purldb.yml rename to docker-compose.common.yml index 5fbce4bf..ca689d00 100644 --- a/docker-compose_purldb.yml +++ b/docker-compose.common.yml @@ -120,25 +120,6 @@ services: - db - web - nginx: - image: nginx - labels: - - "traefik.enable=true" - - "traefik.http.routers.web.rule= - Host(`127.0.0.1`) - || Host(`localhost`)" - - "traefik.http.routers.web.entrypoints=web" - volumes: - - ./etc/nginx/conf.d/:/etc/nginx/conf.d/ - - static:/var/purldb/static/ - depends_on: - - web - -networks: - default: - name: purldb - external: true - volumes: db_data: static: diff --git a/docker-compose_matchcodeio.yml b/docker-compose.matchcodeio.yml similarity index 100% rename from docker-compose_matchcodeio.yml rename to docker-compose.matchcodeio.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..ba81d2f9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,35 @@ +version: "3.7" + +include: + - docker-compose.common.yml + +services: + traefik: + image: "traefik:v2.10" + container_name: "traefik" + hostname: "traefik" + ports: + - "80:80" + - "443:443" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "./traefik.yml:/traefik.yml:ro" + + nginx: + image: nginx + labels: + - "traefik.enable=true" + - "traefik.http.routers.web.rule= + Host(`127.0.0.1`) + || Host(`localhost`)" + - "traefik.http.routers.web.entrypoints=web" + volumes: + - ./etc/nginx/conf.d/:/etc/nginx/conf.d/ + - static:/var/purldb/static/ + depends_on: + - web + +networks: + default: + name: purldb + external: true diff --git a/docker-compose_purldb_public.yml b/docker-compose_purldb_public.yml deleted file mode 100644 index 242243d8..00000000 --- a/docker-compose_purldb_public.yml +++ /dev/null @@ -1,111 +0,0 @@ -version: "3" - -services: - db: - image: postgres:13 - env_file: - - docker_purldb.env - volumes: - - db_data:/var/lib/postgresql/data/ - - web: - build: . - command: sh -c " - python manage_purldb_public.py migrate && - python manage_purldb_public.py collectstatic --no-input --verbosity 0 --clear && - gunicorn purldb_public_project.wsgi:application --bind :8000 --timeout 600 --workers 8" - env_file: - - docker_purldb.env - expose: - - 8000 - volumes: - - /etc/purldb/:/etc/purldb/ - - static:/var/purldb/static/ - depends_on: - - db - - visitor: - build: . - command: sh -c " - wait-for-it web:8000 -- python manage_purldb_public.py seed && - python manage_purldb_public.py run_visit --ignore-robots --ignore-throttle" - env_file: - - docker_purldb.env - volumes: - - /etc/purldb/:/etc/purldb/ - profiles: - - visit_and_map - depends_on: - - db - - web # Ensure that potential db migrations run first - - mapper: - build: . - command: wait-for-it web:8000 -- python manage_purldb_public.py run_map - env_file: - - docker_purldb.env - volumes: - - /etc/purldb/:/etc/purldb/ - profiles: - - visit_and_map - depends_on: - - db - - web # Ensure that potential db migrations run first - - clearsync: - build: . - command: wait-for-it web:8000 -- clearsync --save-to-db --verbose -n 3 - env_file: - - docker_purldb.env - volumes: - - /etc/purldb/:/etc/purldb/ - profiles: - - clearsync - depends_on: - - db - - web # Ensure that potential db migrations run first - - clearindex: - build: . - command: wait-for-it web:8000 -- python manage_purldb_public.py run_clearindex - profiles: - - clearsync - depends_on: - - db - - web # Ensure that potential db migrations run first - - priority_queue: - build: . - command: wait-for-it web:8000 -- python manage_purldb_public.py priority_queue - env_file: - - docker_purldb.env - volumes: - - /etc/purldb/:/etc/purldb/ - profiles: - - priority_queue - depends_on: - - db - - web - - nginx: - image: nginx - labels: - - "traefik.enable=true" - - "traefik.http.routers.web.rule= - Host(`127.0.0.1`) - || Host(`localhost`)" - - "traefik.http.routers.web.entrypoints=web" - volumes: - - ./etc/nginx/conf.d/:/etc/nginx/conf.d/ - - static:/var/purldb/static/ - depends_on: - - web - -networks: - default: - name: purldb - external: true - -volumes: - db_data: - static: diff --git a/docker-compose_traefik.yml b/docker-compose_traefik.yml deleted file mode 100644 index 76e706bc..00000000 --- a/docker-compose_traefik.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3.7" - -services: - traefik: - image: "traefik:v2.10" - container_name: "traefik" - hostname: "traefik" - ports: - - "80:80" - - "443:443" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" - - "./traefik.yml:/traefik.yml:ro" - -networks: - default: - name: purldb - external: true From 983ec4cb7a7fc2973a56e00a24f86e44f6be8b72 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 18 Apr 2024 16:38:48 -0700 Subject: [PATCH 2/3] Remove references to old scan queue commands in readme #391 * Remove manage_purldb_public.py Signed-off-by: Jono Yang --- README.rst | 24 +++--------------------- docker-compose.common.yml | 2 ++ docker-compose.yml | 22 +++++++++++----------- manage_purldb_public.py | 19 ------------------- 4 files changed, 16 insertions(+), 51 deletions(-) delete mode 100644 manage_purldb_public.py diff --git a/README.rst b/README.rst index 4a94f39a..90eef783 100644 --- a/README.rst +++ b/README.rst @@ -24,7 +24,7 @@ Installation Requirements ############ * Debian-based Linux distribution -* Python 3.8 or later +* Python 3.11 or later * Postgres 13 * git * scancode-toolkit runtime dependencies (https://scancode-toolkit.readthedocs.io/en/stable/getting-started/install.html#install-prerequisites) @@ -89,23 +89,6 @@ variables will have to be set for the scan queue commands to work: SCANCODEIO_URL= SCANCODEIO_API_KEY= -The scan queue is run using two commands: -:: - - make request_scans - -``request_scans`` will send a Package scan request to a configured ScanCode.io -instance. ScanCode.io will download, extract, and scan the files of the -requested Package. -:: - - make process_scans - -``process_scans`` will poll ScanCode.io for the status of the Package scans -requested by ``request_scans``. When a Package scan on ScanCode.io is ready, -``process_scans`` will use that data to create Resources and populate the -MatchCode directory fingerprint indices. - Package Resource data can also be gathered by running ClearCode, where Package scan data from clearlydefined is collected and its results are used to create Packages and Resources. @@ -206,9 +189,8 @@ All requests to the host go to the PurlDB service, but requests that go to the To run PurlDB and Matchcode.io with Docker: :: - docker compose -f docker-compose_traefik.yml up -d - docker compose -f docker-compose_purldb.yml up -d - docker compose -f docker-compose_matchcodeio.yml up -d + docker compose -f docker-compose.yml up -d + docker compose -f docker-compose.matchcodeio.yml up -d Funding ------- diff --git a/docker-compose.common.yml b/docker-compose.common.yml index ca689d00..86ab9968 100644 --- a/docker-compose.common.yml +++ b/docker-compose.common.yml @@ -76,6 +76,8 @@ services: clearindex: build: . command: wait-for-it web:8000 -- python manage_purldb.py run_clearindex + env_file: + - docker_purldb.env profiles: - clearsync depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index ba81d2f9..ed5744f3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,17 +4,6 @@ include: - docker-compose.common.yml services: - traefik: - image: "traefik:v2.10" - container_name: "traefik" - hostname: "traefik" - ports: - - "80:80" - - "443:443" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" - - "./traefik.yml:/traefik.yml:ro" - nginx: image: nginx labels: @@ -29,6 +18,17 @@ services: depends_on: - web + traefik: + image: "traefik:v2.10" + container_name: "traefik" + hostname: "traefik" + ports: + - "80:80" + - "443:443" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:ro" + - "./traefik.yml:/traefik.yml:ro" + networks: default: name: purldb diff --git a/manage_purldb_public.py b/manage_purldb_public.py deleted file mode 100644 index ba7f0bbb..00000000 --- a/manage_purldb_public.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# purldb is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/purldb for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -import sys - - -if __name__ == '__main__': - from django.core.management import execute_from_command_line - - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'purldb_public_project.settings') - execute_from_command_line(sys.argv) From 1073c83ba5c421a2c5be15239ea5e22a6d11c496 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 18 Apr 2024 18:29:17 -0700 Subject: [PATCH 3/3] Add missing migration Signed-off-by: Jono Yang --- ...0085_alter_packagerelation_relationship.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 packagedb/migrations/0085_alter_packagerelation_relationship.py diff --git a/packagedb/migrations/0085_alter_packagerelation_relationship.py b/packagedb/migrations/0085_alter_packagerelation_relationship.py new file mode 100644 index 00000000..2dcf1144 --- /dev/null +++ b/packagedb/migrations/0085_alter_packagerelation_relationship.py @@ -0,0 +1,28 @@ +# Generated by Django 5.0.4 on 2024-04-19 01:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "packagedb", + "0084_alter_resource_file_type_alter_resource_mime_type_and_more", + ), + ] + + operations = [ + migrations.AlterField( + model_name="packagerelation", + name="relationship", + field=models.CharField( + choices=[ + ("source_package", "Source Package"), + ("binary_package", "Binary Package"), + ], + help_text='Relationship between the from and to package URLs such as "source_package" when a package is the source code package for another package.', + max_length=30, + ), + ), + ]