From 228825fb32b846653541e09941a8b7d5d1d9535a Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Sun, 23 Aug 2026 11:59:48 -0700 Subject: [PATCH 1/3] Add development database dump setup --- README.md | 4 + apps/gcd/management/__init__.py | 1 + apps/gcd/management/commands/__init__.py | 1 + .../commands/seed_development_data.py | 25 +++ apps/gcd/tests/test_dev_environment_core.py | 12 ++ apps/gcd/tests/test_seed_development_data.py | 19 ++ bin/dev | 196 ++++++++++++++++++ docs/development/CORE_SETUP.md | 53 +++++ 8 files changed, 311 insertions(+) create mode 100644 apps/gcd/management/__init__.py create mode 100644 apps/gcd/management/commands/__init__.py create mode 100644 apps/gcd/management/commands/seed_development_data.py create mode 100644 apps/gcd/tests/test_seed_development_data.py diff --git a/README.md b/README.md index bca327da9..9934b34e7 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,10 @@ http://groups.google.com/group/gcd-tech/ See [the core development environment guide](docs/development/CORE_SETUP.md) for the supported Docker and Docker-free local setup. +For the default Docker setup with deterministic development data, run +`./bin/dev setup`. Contributors who have manually downloaded an authenticated +GCD catalog archive can use `./bin/dev setup --dump ~/Downloads/current.zip`. + You can find manual instructions for various platforms using virtual environments for python in the docs directory [GCD Docs](https://github.com/GrandComicsDatabase/gcd-django/tree/beta/docs) but they aren't necessarily up to date. As of September 2023 they should work. diff --git a/apps/gcd/management/__init__.py b/apps/gcd/management/__init__.py new file mode 100644 index 000000000..3ad7cb0df --- /dev/null +++ b/apps/gcd/management/__init__.py @@ -0,0 +1 @@ +"""Management-command package for the GCD application.""" diff --git a/apps/gcd/management/commands/__init__.py b/apps/gcd/management/commands/__init__.py new file mode 100644 index 000000000..c323cd041 --- /dev/null +++ b/apps/gcd/management/commands/__init__.py @@ -0,0 +1 @@ +"""GCD management commands.""" diff --git a/apps/gcd/management/commands/seed_development_data.py b/apps/gcd/management/commands/seed_development_data.py new file mode 100644 index 000000000..2e2f941b0 --- /dev/null +++ b/apps/gcd/management/commands/seed_development_data.py @@ -0,0 +1,25 @@ +"""Create the small, deterministic dataset used for local development.""" + +from django.core.management import BaseCommand, call_command +from django.db import transaction + +from apps.stats.models import CountStats + + +class Command(BaseCommand): + """Load local accounts and the statistics required by edit workflows.""" + + help = ( + 'Load deterministic development accounts and initialize global ' + 'catalog statistics.' + ) + + def handle(self, *args, **options): + """Seed data safely on a new database or refresh it on an existing one.""" + with transaction.atomic(): + call_command('loaddata', 'users', verbosity=options['verbosity']) + CountStats.objects.init_stats() + + self.stdout.write( + self.style.SUCCESS('Development accounts and catalog statistics are ready.') + ) diff --git a/apps/gcd/tests/test_dev_environment_core.py b/apps/gcd/tests/test_dev_environment_core.py index 9f45daacf..b08cca524 100644 --- a/apps/gcd/tests/test_dev_environment_core.py +++ b/apps/gcd/tests/test_dev_environment_core.py @@ -94,6 +94,8 @@ def test_dev_launcher_documents_supported_commands(): assert result.returncode == 0 assert './bin/dev up' in result.stdout + assert './bin/dev setup' in result.stdout + assert './bin/dev setup --dump ~/Downloads/current.zip' in result.stdout assert '--runtime native' in result.stdout assert 'reset --yes' in result.stdout @@ -106,6 +108,16 @@ def test_dev_launcher_refuses_reset_without_explicit_confirmation(): assert '--yes' in result.stderr +def test_dev_launcher_declares_a_confirmation_gated_dump_setup_flow(): + """Full-catalog setup stays a single command without silent replacement.""" + launcher = _read_project_file('bin/dev') + + assert 'setup [--dump ARCHIVE] [--replace --yes]' in launcher + assert 'setup_dump_database' in launcher + assert 'seed_development_data' in launcher + assert 'A non-empty local database will be replaced' in launcher + + def test_dev_launcher_handles_crlf_dotenv_and_native_database_overrides(): """The .env parser supports Windows endings and native DB configuration.""" launcher = _read_project_file('bin/dev') diff --git a/apps/gcd/tests/test_seed_development_data.py b/apps/gcd/tests/test_seed_development_data.py new file mode 100644 index 000000000..b8ad027b4 --- /dev/null +++ b/apps/gcd/tests/test_seed_development_data.py @@ -0,0 +1,19 @@ +"""Tests for the deterministic local-development data command.""" + +from django.contrib.auth.models import User +from django.core.management import call_command + +from apps.indexer.models import Indexer +from apps.stats.models import CountStats + + +def test_seed_development_data_is_idempotent_and_initializes_stats(db): + """Contributors can rerun setup without duplicating accounts or statistics.""" + call_command('seed_development_data') + call_command('seed_development_data') + + assert User.objects.filter(username='admin').count() == 1 + assert User.objects.filter(username='editor').count() == 1 + assert User.objects.filter(username='anon').count() == 1 + assert Indexer.objects.filter(user__username='admin').exists() + assert CountStats.objects.filter(language=None, country=None).count() == 8 diff --git a/bin/dev b/bin/dev index fb0c48514..6f3f40d4f 100755 --- a/bin/dev +++ b/bin/dev @@ -19,6 +19,9 @@ intentional Docker-free path for a locally installed Python 3.13 and MySQL 8. Commands: up Start the application at http://127.0.0.1:8000 + setup [--dump ARCHIVE] [--replace --yes] + Create a ready-to-use local environment, optionally + loading a manually downloaded GCD catalog dump down Stop the Docker stack (native has nothing to stop) test [pytest args] Run the test suite shell Open a Django development shell @@ -30,6 +33,8 @@ Commands: Examples: ./bin/dev up + ./bin/dev setup + ./bin/dev setup --dump ~/Downloads/current.zip ./bin/dev test apps/gcd/tests ./bin/dev manage createsuperuser ./bin/dev --runtime native up @@ -119,6 +124,168 @@ native_database_name() { "$PYTHON_BIN" -c 'import django; django.setup(); from django.conf import settings; print(settings.DATABASES["default"]["NAME"])' } +mysql_root() { + compose exec -T db sh -c 'mysql -uroot -p"$MYSQL_ROOT_PASSWORD"' +} + +mysql_root_without_headers() { + compose exec -T db sh -c 'mysql -N -uroot -p"$MYSQL_ROOT_PASSWORD"' +} + +mysql_root_query() { + printf '%s\n' "$1" | mysql_root_without_headers +} + +validate_setup_database_name() { + if [[ ! "$MYSQL_DATABASE" =~ ^gcd_dev(_[a-z0-9_]+)?$ ]] && \ + [[ "$MYSQL_DATABASE" != "gcd_django_dev" ]]; then + echo "setup only permits a local development database named gcd_dev, gcd_django_dev, or gcd_dev_* (got $MYSQL_DATABASE)." >&2 + exit 2 + fi +} + +database_has_user_data() { + local table_count + table_count="$(mysql_root_query "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${MYSQL_DATABASE}';")" + [[ "$table_count" != "0" ]] +} + +confirm_setup_replacement() { + local replace="$1" yes="$2" response + if ! database_has_user_data; then + return + fi + if [[ "$replace" == true && "$yes" == true ]]; then + return + fi + if [[ ! -t 0 ]]; then + echo "A non-empty local database will be replaced. Rerun with --replace --yes to confirm." >&2 + exit 2 + fi + read -r -p "A non-empty local database will be replaced. Continue? [y/N] " response + if [[ "$response" != "y" && "$response" != "Y" ]]; then + echo "Setup cancelled." + exit 0 + fi +} + +fresh_docker_database() { + compose down --volumes + compose build web + compose up -d --wait db + compose run --rm --no-deps web python manage.py migrate --noinput +} + +dump_sql_entry() { + local archive="$1" entry sql_entry="" + if ! command -v unzip >/dev/null 2>&1; then + echo "setup --dump needs the unzip command to read a ZIP archive." >&2 + exit 1 + fi + if ! unzip -t -- "$archive" >/dev/null; then + echo "The dump archive failed its integrity check: $archive" >&2 + exit 1 + fi + while IFS= read -r entry; do + if [[ -n "$sql_entry" ]]; then + echo "The dump archive must contain exactly one .sql file." >&2 + exit 1 + fi + sql_entry="$entry" + done < <(unzip -Z1 -- "$archive" | awk '/\.sql$/ { print }') + if [[ -z "$sql_entry" ]]; then + echo "The dump archive must contain exactly one .sql file." >&2 + exit 1 + fi + printf '%s\n' "$sql_entry" +} + +stream_dump_to_stage() { + local archive="$1" sql_entry + case "$archive" in + *.zip) + sql_entry="$(dump_sql_entry "$archive")" + { + printf '%s\n' 'CREATE DATABASE gcd_dev_import_stage CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;' + printf '%s\n' 'USE gcd_dev_import_stage;' + unzip -p -- "$archive" "$sql_entry" + } | mysql_root + ;; + *.sql) + { + printf '%s\n' 'CREATE DATABASE gcd_dev_import_stage CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;' + printf '%s\n' 'USE gcd_dev_import_stage;' + cat -- "$archive" + } | mysql_root + ;; + *) + echo "The dump must be a .zip archive or .sql file." >&2 + exit 2 + ;; + esac +} + +validate_dump_schema() { + local missing_columns incompatible_columns + missing_columns="$(mysql_root_query "SELECT CONCAT(source_columns.table_name, '.', source_columns.column_name) FROM information_schema.columns AS source_columns LEFT JOIN information_schema.columns AS target_columns ON target_columns.table_schema = '${MYSQL_DATABASE}' AND target_columns.table_name = source_columns.table_name AND target_columns.column_name = source_columns.column_name WHERE source_columns.table_schema = 'gcd_dev_import_stage' AND source_columns.table_name REGEXP '^(gcd|stddata)_' AND target_columns.column_name IS NULL ORDER BY source_columns.table_name, source_columns.ordinal_position;")" + if [[ -n "$missing_columns" ]]; then + echo "The dump has columns that are not present in this source checkout:" >&2 + echo "$missing_columns" >&2 + exit 1 + fi + incompatible_columns="$(mysql_root_query "SELECT CONCAT(source_columns.table_name, '.', source_columns.column_name, ' (dump ', source_columns.column_type, ', target ', target_columns.column_type, ')') FROM information_schema.columns AS source_columns JOIN information_schema.columns AS target_columns ON target_columns.table_schema = '${MYSQL_DATABASE}' AND target_columns.table_name = source_columns.table_name AND target_columns.column_name = source_columns.column_name WHERE source_columns.table_schema = 'gcd_dev_import_stage' AND source_columns.table_name REGEXP '^(gcd|stddata)_' AND (source_columns.is_nullable <> target_columns.is_nullable OR (source_columns.column_type <> target_columns.column_type AND NOT (source_columns.column_type = 'datetime' AND target_columns.column_type = 'datetime(6)') AND NOT (source_columns.column_type = 'tinyint(1)' AND target_columns.column_type = 'int'))) ORDER BY source_columns.table_name, source_columns.ordinal_position;")" + if [[ -n "$incompatible_columns" ]]; then + echo "The dump schema is incompatible with this source checkout:" >&2 + echo "$incompatible_columns" >&2 + exit 1 + fi +} + +copy_dump_catalog() { + local catalog_copy_sql auto_increment_sql + catalog_copy_sql="$(mysql_root_query "SELECT CONCAT('DELETE FROM ', CHAR(96), '${MYSQL_DATABASE}', CHAR(96), '.', CHAR(96), table_name, CHAR(96), '; INSERT INTO ', CHAR(96), '${MYSQL_DATABASE}', CHAR(96), '.', CHAR(96), table_name, CHAR(96), ' (', GROUP_CONCAT(CONCAT(CHAR(96), column_name, CHAR(96)) ORDER BY ordinal_position SEPARATOR ','), ') SELECT ', GROUP_CONCAT(CONCAT(CHAR(96), column_name, CHAR(96)) ORDER BY ordinal_position SEPARATOR ','), ' FROM ', CHAR(96), 'gcd_dev_import_stage', CHAR(96), '.', CHAR(96), table_name, CHAR(96), ';') FROM information_schema.columns WHERE table_schema = 'gcd_dev_import_stage' AND table_name REGEXP '^(gcd|stddata)_' GROUP BY table_name ORDER BY table_name;")" + auto_increment_sql="$(mysql_root_query "SELECT CONCAT('ALTER TABLE ', CHAR(96), '${MYSQL_DATABASE}', CHAR(96), '.', CHAR(96), table_name, CHAR(96), ' AUTO_INCREMENT = 1;') FROM information_schema.columns WHERE table_schema = 'gcd_dev_import_stage' AND table_name REGEXP '^(gcd|stddata)_' AND extra LIKE '%auto_increment%' GROUP BY table_name ORDER BY table_name;")" + { + printf '%s\n' 'SET FOREIGN_KEY_CHECKS=0;' + printf '%s\n' "$catalog_copy_sql" + printf '%s\n' "$auto_increment_sql" + printf '%s\n' 'SET FOREIGN_KEY_CHECKS=1;' + } | mysql_root +} + +setup_dump_database() { + local archive="$1" + if [[ ! -f "$archive" ]]; then + echo "Dump archive not found: $archive" >&2 + exit 2 + fi + mysql_root_query 'DROP DATABASE IF EXISTS gcd_dev_import_stage;' + echo "Importing the catalog dump into a temporary local staging database..." + stream_dump_to_stage "$archive" + validate_dump_schema + echo "Copying compatible catalog data into the migrated development database..." + copy_dump_catalog + mysql_root_query 'DROP DATABASE gcd_dev_import_stage;' +} + +seed_docker_development_data() { + compose run --rm --no-deps web python manage.py seed_development_data +} + +setup_docker_environment() { + local archive="$1" replace="$2" yes="$3" + validate_setup_database_name + compose up -d --wait db + confirm_setup_replacement "$replace" "$yes" + fresh_docker_database + if [[ -n "$archive" ]]; then + setup_dump_database "$archive" + fi + seed_docker_development_data + compose up --build --wait + echo "GCD is available at http://127.0.0.1:${GCD_WEB_PORT}" +} + runtime="docker" if [[ "${1:-}" == "--runtime" ]]; then runtime="${2:-}" @@ -136,6 +303,35 @@ set_defaults case "$command" in help|-h|--help) usage ;; + setup) + setup_archive="" + setup_replace=false + setup_yes=false + while [[ $# -gt 0 ]]; do + case "$1" in + --dump) + if [[ $# -lt 2 ]]; then + echo "Usage: ./bin/dev setup --dump ARCHIVE [--replace --yes]" >&2 + exit 2 + fi + setup_archive="$2" + shift 2 + ;; + --replace) setup_replace=true; shift ;; + --yes) setup_yes=true; shift ;; + *) + echo "Unknown setup option: $1" >&2 + exit 2 + ;; + esac + done + if [[ "$runtime" != "docker" ]]; then + echo "setup currently supports the default Docker runtime. Use ./bin/dev --runtime native up for Docker-free development." >&2 + exit 2 + fi + require_docker + setup_docker_environment "$setup_archive" "$setup_replace" "$setup_yes" + ;; up) if [[ "$runtime" == "docker" ]]; then require_docker diff --git a/docs/development/CORE_SETUP.md b/docs/development/CORE_SETUP.md index ae2b49fd5..8a6c5e2a7 100644 --- a/docs/development/CORE_SETUP.md +++ b/docs/development/CORE_SETUP.md @@ -17,10 +17,63 @@ The application is available at . MySQL is available only on `127.0.0.1:3308` from the local machine. The database data lives in the Docker volume named `gcd-django-dev_mysql_data`. +## Ready-to-use development data + +For a small, deterministic local dataset, use one command instead of running +migrations, fixtures, and account setup separately: + +```bash +./bin/dev setup +``` + +It creates the development database, applies migrations, loads local +development accounts, initializes global statistics, and starts the +application. + +The seeded accounts are deliberately public local-development fixtures. They +work only in this local database and must never be used for beta or production: + +| Username | Password | Intended use | +| --- | --- | --- | +| `admin` | `admin` | Local Django superuser and administrator | +| `editor` | `editme` | Editor, indexer, and member workflow testing | +| `dexter_1234` | `test` | Standard indexer workflow testing | +| `anon` | — | Anonymous fixture account; it cannot log in | + +### Full GCD catalog dump + +The full catalog dump is optional. Download it manually from + after accepting the GCD download terms, then +run: + +```bash +./bin/dev setup --dump ~/Downloads/current.zip +``` + +The command handles everything else: it validates the archive, restores it to +a temporary local staging database, creates a clean migrated development +database, checks compatibility, copies catalog data, initializes local +accounts/statistics, and starts the site. It does not need a database name, +manual SQL command, fake migration, or separate seed step from the +contributor. + +If a local database already exists, `setup` asks once before replacing it. For +non-interactive use, supply the explicit confirmation: + +```bash +./bin/dev setup --dump ~/Downloads/current.zip --replace --yes +``` + +The full-dump workflow is currently supported by the default Docker runtime. +It needs approximately 12 GB of free Docker storage while the temporary +staging database and final development database coexist. The public dump does +not contain uploaded cover/image files; development uses fake images instead. + Useful commands: ```bash ./bin/dev doctor +./bin/dev setup ./bin/dev test ./bin/dev manage createsuperuser ./bin/dev logs web From 88f0605efbb68323bcb19b70aa2528c6ecd20a6f Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Tue, 1 Sep 2026 17:13:10 -0700 Subject: [PATCH 2/3] Seed deterministic development catalog data --- README.md | 5 +- .../commands/seed_development_data.py | 326 +++++++++++++++++- apps/gcd/tests/test_seed_development_data.py | 38 ++ docs/development/CORE_SETUP.md | 16 +- 4 files changed, 372 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 9934b34e7..7e7eb9070 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,8 @@ For the default Docker setup with deterministic development data, run `./bin/dev setup`. Contributors who have manually downloaded an authenticated GCD catalog archive can use `./bin/dev setup --dump ~/Downloads/current.zip`. -You can find manual instructions for various platforms using virtual environments for python in the docs directory -[GCD Docs](https://github.com/GrandComicsDatabase/gcd-django/tree/beta/docs) but they aren't -necessarily up to date. As of September 2023 they should work. +The core guide is the canonical setup path. Platform-specific notes in the +`docs` directory are retained for reference and may describe older workflows. ## Workflow diff --git a/apps/gcd/management/commands/seed_development_data.py b/apps/gcd/management/commands/seed_development_data.py index 2e2f941b0..d5ad53f9c 100644 --- a/apps/gcd/management/commands/seed_development_data.py +++ b/apps/gcd/management/commands/seed_development_data.py @@ -1,25 +1,343 @@ """Create the small, deterministic dataset used for local development.""" +import base64 + +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType +from django.core.files.base import ContentFile from django.core.management import BaseCommand, call_command from django.db import transaction +from apps.gcd.models import ( + Character, + CharacterNameDetail, + CharacterRole, + Cover, + Creator, + CreatorNameDetail, + CreditType, + Feature, + FeatureNameDetail, + FeatureType, + Group, + GroupNameDetail, + Image, + ImageType, + Issue, + Publisher, + Reprint, + Series, + Story, + StoryCharacter, + StoryCredit, + StoryType, + Universe, +) +from apps.oi import states +from apps.oi.models import CTYPES, Changeset, ChangesetComment +from apps.stddata.models import Country, Language, Script from apps.stats.models import CountStats +SAMPLE_PREFIX = '[GCD DEV]' +SAMPLE_COMMENT = ( + '[GCD DEV] Seeded sample change history for local development. ✅' +) +SAMPLE_PNG = base64.b64decode( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk' + 'YAAAAAYAAjCB0C8AAAAASUVORK5CYII=' +) + + class Command(BaseCommand): - """Load local accounts and the statistics required by edit workflows.""" + """Load local accounts, sample catalog data, and edit-workflow state.""" help = ( - 'Load deterministic development accounts and initialize global ' - 'catalog statistics.' + 'Load deterministic development accounts, sample catalog data, and ' + 'global catalog statistics.' ) def handle(self, *args, **options): """Seed data safely on a new database or refresh it on an existing one.""" with transaction.atomic(): call_command('loaddata', 'users', verbosity=options['verbosity']) + self._seed_catalog() + self._seed_change_history() CountStats.objects.init_stats() self.stdout.write( - self.style.SUCCESS('Development accounts and catalog statistics are ready.') + self.style.SUCCESS( + 'Development accounts, sample catalog data, and statistics are ready.' + ) + ) + + def _seed_catalog(self): + """Create a compact, relationship-rich catalog sample idempotently.""" + country = Country.objects.filter(code='us').first() or Country.objects.first() + language = (Language.objects.filter(code='en').first() + or Language.objects.first()) + if country is None or language is None: + raise RuntimeError('Country and language reference data are required.') + + publisher, _ = Publisher.objects.get_or_create( + name=f'{SAMPLE_PREFIX} Example Comics', + defaults={'notes': 'Deterministic publisher for local development.', + 'country': country}, + ) + series, _ = Series.objects.get_or_create( + publisher=publisher, + name=f'{SAMPLE_PREFIX} Adventures', + defaults={ + 'sort_name': f'{SAMPLE_PREFIX} Adventures', + 'notes': 'Deterministic series for local development.', + 'year_began': 2024, + 'publication_dates': '2024-present', + 'tracking_notes': '', + 'country': country, + 'language': language, + 'is_comics_publication': True, + 'is_current': True, + }, + ) + issue, _ = Issue.objects.get_or_create( + series=series, + sort_code=1, + defaults={ + 'number': '1', + 'title': f'{SAMPLE_PREFIX} First Flight', + 'volume': '1', + 'isbn': '', + 'valid_isbn': '', + 'variant_name': '', + 'barcode': '', + 'publication_date': '2024', + 'key_date': '2024-01-01', + 'on_sale_date': '2024-01-01', + 'indicia_frequency': 'monthly', + 'price': '$3.99', + 'editing': '', + 'notes': 'Deterministic issue for local development.', + 'indicia_printer_sourced_by': '', + 'is_indexed': 10, + }, + ) + variant, _ = Issue.objects.get_or_create( + series=series, + sort_code=2, + defaults={ + 'number': '1', + 'title': f'{SAMPLE_PREFIX} First Flight', + 'volume': '1', + 'isbn': '', + 'valid_isbn': '', + 'variant_name': 'Direct Market Variant', + 'barcode': '', + 'publication_date': '2024', + 'key_date': '2024-01-01', + 'on_sale_date': '2024-01-01', + 'indicia_frequency': 'monthly', + 'price': '$3.99', + 'editing': '', + 'notes': 'Deterministic variant for local development.', + 'indicia_printer_sourced_by': '', + 'is_indexed': 10, + 'variant_of': issue, + 'variant_cover_status': 3, + }, + ) + + feature_type, _ = FeatureType.objects.get_or_create(name='story') + feature, _ = Feature.objects.get_or_create( + name=f'{SAMPLE_PREFIX} Feature', + language=language, + defaults={ + 'sort_name': f'{SAMPLE_PREFIX} Feature', + 'genre': 'superhero', + 'feature_type': feature_type, + 'description': 'Deterministic feature for local development.', + 'notes': '', + }, + ) + feature_name, _ = FeatureNameDetail.objects.get_or_create( + feature=feature, + name=f'{SAMPLE_PREFIX} Feature', + defaults={'is_official_name': True}, + ) + story_type, _ = StoryType.objects.get_or_create( + name='cartoon', defaults={'sort_code': 7} + ) + story, story_created = Story.objects.get_or_create( + issue=issue, + sequence_number=0, + defaults={ + 'title': f'{SAMPLE_PREFIX} First Flight Story', + 'feature': feature.name, + 'type': story_type, + 'script': '', + 'pencils': '', + 'inks': '', + 'colors': '', + 'letters': '', + 'editing': '', + 'job_number': '', + 'genre': '', + 'characters': '', + 'synopsis': 'Deterministic story for local development.', + 'reprint_notes': '', + 'notes': '', + }, + ) + if not story_created and (story.type_id != story_type.id or + story.title != f'{SAMPLE_PREFIX} First Flight Story'): + story.type = story_type + story.title = f'{SAMPLE_PREFIX} First Flight Story' + story.save(update_fields=['type', 'title']) + story.feature_object.add(feature) + story.feature_name.add(feature_name) + + creator, _ = Creator.objects.get_or_create( + gcd_official_name=f'{SAMPLE_PREFIX} Alex Example', + defaults={ + 'bio': 'Deterministic creator for local development.', + 'notes': '', + 'sort_name': f'{SAMPLE_PREFIX} Alex Example', + 'birth_province': '', + 'birth_city': '', + 'death_province': '', + 'death_city': '', + }, + ) + script = Script.objects.filter(number=37).first() + if script is None: + script = Script.objects.filter(code='latn').first() + if script is None: + script = Script.objects.create(number=37, code='latn', name='Latin') + creator_name, _ = CreatorNameDetail.objects.get_or_create( + creator=creator, + name=f'{SAMPLE_PREFIX} Alex Example', + defaults={'is_official_name': True, 'in_script': script}, + ) + credit_type, _ = CreditType.objects.get_or_create( + name='script', defaults={'sort_code': 1} + ) + StoryCredit.objects.get_or_create( + story=story, + creator=creator_name, + credit_type=credit_type, + defaults={ + 'signed_as': '', + 'credited_as': creator_name.name, + 'sourced_by': '', + 'credit_name': '', + 'is_credited': True, + }, + ) + + universe, _ = Universe.objects.get_or_create( + multiverse=f'{SAMPLE_PREFIX} Multiverse', + name=f'{SAMPLE_PREFIX} Main Universe', + designation='Earth-DEV', + defaults={ + 'description': 'Deterministic universe for local development.', + 'notes': '', + 'year_first_published': 2024, + }, + ) + character, _ = Character.objects.get_or_create( + name=f'{SAMPLE_PREFIX} Captain Example', + defaults={ + 'sort_name': f'{SAMPLE_PREFIX} Captain Example', + 'disambiguation': 'development sample', + 'universe': universe, + 'language': language, + 'description': 'Deterministic character for local development.', + 'notes': '', + }, + ) + character_name, _ = CharacterNameDetail.objects.get_or_create( + character=character, + name=f'{SAMPLE_PREFIX} Captain Example', + defaults={'is_official_name': True}, + ) + group, _ = Group.objects.get_or_create( + name=f'{SAMPLE_PREFIX} Example League', + defaults={ + 'sort_name': f'{SAMPLE_PREFIX} Example League', + 'disambiguation': 'development sample', + 'universe': universe, + 'language': language, + 'description': 'Deterministic group for local development.', + 'notes': '', + }, + ) + group_name, _ = GroupNameDetail.objects.get_or_create( + group=group, + name=f'{SAMPLE_PREFIX} Example League', + defaults={'is_official_name': True}, + ) + role, _ = CharacterRole.objects.get_or_create( + name='featured', defaults={'sort_code': 3} + ) + appearance, _ = StoryCharacter.objects.get_or_create( + story=story, + character=character_name, + defaults={'universe': universe, 'role': role, 'notes': ''}, + ) + appearance.group.add(group) + appearance.group_name.add(group_name) + story.universe.add(universe) + Reprint.objects.get_or_create( + origin=story, + target=None, + origin_issue=issue, + target_issue=variant, + defaults={'notes': 'Deterministic issue reprint for local development.'}, + ) + + image_type, _ = ImageType.objects.get_or_create( + name=f'{SAMPLE_PREFIX} Cover Image', + defaults={ + 'description': 'Deterministic cover image for local development.' + }, + ) + issue_content_type = ContentType.objects.get_for_model(Issue) + for covered_issue in (issue, variant): + Cover.objects.get_or_create(issue=covered_issue) + image, _ = Image.objects.get_or_create( + content_type=issue_content_type, + object_id=covered_issue.id, + type=image_type, + ) + if not image.image_file: + image.image_file.save( + f'gcd-dev-cover-{covered_issue.sort_code}.png', + ContentFile(SAMPLE_PNG), + save=True, + ) + + series.set_first_last_issues() + series.issue_count = series.active_issues().count() + series.save(update_fields=['issue_count']) + publisher.issue_count = Issue.objects.filter(series__publisher=publisher, + deleted=False).count() + publisher.series_count = publisher.active_series().count() + publisher.save(update_fields=['issue_count', 'series_count']) + + def _seed_change_history(self): + """Create one approved changeset and emoji-bearing comment sample.""" + admin = User.objects.get(username='admin') + comment = ChangesetComment.objects.filter(text=SAMPLE_COMMENT).first() + if comment: + return + changeset = Changeset.objects.create( + state=states.APPROVED, + indexer=admin, + change_type=CTYPES['series'], + ) + ChangesetComment.objects.create( + commenter=admin, + changeset=changeset, + text=SAMPLE_COMMENT, + old_state=states.PENDING, + new_state=states.APPROVED, ) diff --git a/apps/gcd/tests/test_seed_development_data.py b/apps/gcd/tests/test_seed_development_data.py index b8ad027b4..10cb91d42 100644 --- a/apps/gcd/tests/test_seed_development_data.py +++ b/apps/gcd/tests/test_seed_development_data.py @@ -3,7 +3,24 @@ from django.contrib.auth.models import User from django.core.management import call_command +from apps.gcd.models import ( + Character, + Cover, + Creator, + Feature, + Group, + Image, + Issue, + Publisher, + Reprint, + Series, + Story, + StoryCharacter, + StoryCredit, + Universe, +) from apps.indexer.models import Indexer +from apps.oi.models import ChangesetComment from apps.stats.models import CountStats @@ -17,3 +34,24 @@ def test_seed_development_data_is_idempotent_and_initializes_stats(db): assert User.objects.filter(username='anon').count() == 1 assert Indexer.objects.filter(user__username='admin').exists() assert CountStats.objects.filter(language=None, country=None).count() == 8 + + publisher = Publisher.objects.get(name='[GCD DEV] Example Comics') + series = Series.objects.get(name='[GCD DEV] Adventures', publisher=publisher) + issue = Issue.objects.get(series=series, sort_code=1) + variant = Issue.objects.get(series=series, sort_code=2) + story = Story.objects.get(issue=issue, sequence_number=0) + + assert variant.variant_of == issue + assert Feature.objects.filter(name='[GCD DEV] Feature').exists() + assert Creator.objects.filter(gcd_official_name='[GCD DEV] Alex Example').exists() + assert StoryCredit.objects.filter(story=story).exists() + assert Universe.objects.filter(designation='Earth-DEV').exists() + assert Character.objects.filter(name='[GCD DEV] Captain Example').exists() + assert Group.objects.filter(name='[GCD DEV] Example League').exists() + assert StoryCharacter.objects.filter(story=story).exists() + assert Reprint.objects.filter(origin=story, target_issue=variant).exists() + assert Cover.objects.filter(issue__in=(issue, variant)).count() == 2 + assert Image.objects.filter(object_id__in=(issue.id, variant.id)).exclude( + image_file='' + ).count() == 2 + assert ChangesetComment.objects.filter(text__contains='[GCD DEV]').count() == 1 diff --git a/docs/development/CORE_SETUP.md b/docs/development/CORE_SETUP.md index 8a6c5e2a7..5fa311988 100644 --- a/docs/development/CORE_SETUP.md +++ b/docs/development/CORE_SETUP.md @@ -1,9 +1,8 @@ # Core development environment This is the supported one-clone setup for working on `gcd-django`. It runs the -application with Python 3.13 and MySQL 8.0. Elasticsearch, sample data, image -fixtures, and optional service integrations are intentionally outside this -first setup layer. +application with Python 3.13 and MySQL 8.0. Elasticsearch and optional service +integrations are intentionally outside this first setup layer. ## Docker (default) @@ -27,8 +26,10 @@ migrations, fixtures, and account setup separately: ``` It creates the development database, applies migrations, loads local -development accounts, initializes global statistics, and starts the -application. +development accounts, seeds a small relationship-rich catalog, creates a +dummy cover image and approved change-history comment, initializes global +statistics, and starts the application. The sample records are marked +`[GCD DEV]` so they are easy to find and safe to recreate. The seeded accounts are deliberately public local-development fixtures. They work only in this local database and must never be used for beta or production: @@ -67,7 +68,10 @@ non-interactive use, supply the explicit confirmation: The full-dump workflow is currently supported by the default Docker runtime. It needs approximately 12 GB of free Docker storage while the temporary staging database and final development database coexist. The public dump does -not contain uploaded cover/image files; development uses fake images instead. +not contain uploaded cover/image files; development uses the deterministic fake +image created by `seed_development_data` instead. The dump also does not +replace the local migration ledger, accounts, statistics, or change-history +fixtures. Useful commands: From ce562b76956c9a6aa2d1853e166e3eea205fea1e Mon Sep 17 00:00:00 2001 From: Adam Hernandez Date: Tue, 1 Sep 2026 18:10:21 -0700 Subject: [PATCH 3/3] Validate development dump setup inputs --- apps/gcd/tests/test_dev_environment_core.py | 17 ++++++++++++++ bin/dev | 26 +++++++++++++++++---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/apps/gcd/tests/test_dev_environment_core.py b/apps/gcd/tests/test_dev_environment_core.py index b08cca524..1a788465e 100644 --- a/apps/gcd/tests/test_dev_environment_core.py +++ b/apps/gcd/tests/test_dev_environment_core.py @@ -118,6 +118,23 @@ def test_dev_launcher_declares_a_confirmation_gated_dump_setup_flow(): assert 'A non-empty local database will be replaced' in launcher +def test_dev_launcher_validates_dump_before_replacing_database(): + """Invalid dump paths and extensions cannot trigger a database reset.""" + launcher = _read_project_file('bin/dev') + setup_body = launcher.split('setup_docker_environment() {', 1)[1] + assert 'validate_dump_archive "$archive"' in setup_body + assert setup_body.index('validate_dump_archive "$archive"') < \ + setup_body.index('fresh_docker_database') + assert 'The dump must be a .zip archive or .sql file.' in launcher + + +def test_dev_launcher_allows_long_catalog_column_lists(): + """Dump table-copy SQL is not truncated by MySQL's default limit.""" + launcher = _read_project_file('bin/dev') + assert 'SET SESSION group_concat_max_len = 1000000; SELECT CONCAT' \ + in launcher + + def test_dev_launcher_handles_crlf_dotenv_and_native_database_overrides(): """The .env parser supports Windows endings and native DB configuration.""" launcher = _read_project_file('bin/dev') diff --git a/bin/dev b/bin/dev index 6f3f40d4f..0746355f1 100755 --- a/bin/dev +++ b/bin/dev @@ -136,6 +136,21 @@ mysql_root_query() { printf '%s\n' "$1" | mysql_root_without_headers } +validate_dump_archive() { + local archive="$1" + if [[ ! -f "$archive" ]]; then + echo "Dump archive not found: $archive" >&2 + exit 2 + fi + case "$archive" in + *.zip|*.sql) ;; + *) + echo "The dump must be a .zip archive or .sql file." >&2 + exit 2 + ;; + esac +} + validate_setup_database_name() { if [[ ! "$MYSQL_DATABASE" =~ ^gcd_dev(_[a-z0-9_]+)?$ ]] && \ [[ "$MYSQL_DATABASE" != "gcd_django_dev" ]]; then @@ -202,6 +217,7 @@ dump_sql_entry() { stream_dump_to_stage() { local archive="$1" sql_entry + validate_dump_archive "$archive" case "$archive" in *.zip) sql_entry="$(dump_sql_entry "$archive")" @@ -243,7 +259,7 @@ validate_dump_schema() { copy_dump_catalog() { local catalog_copy_sql auto_increment_sql - catalog_copy_sql="$(mysql_root_query "SELECT CONCAT('DELETE FROM ', CHAR(96), '${MYSQL_DATABASE}', CHAR(96), '.', CHAR(96), table_name, CHAR(96), '; INSERT INTO ', CHAR(96), '${MYSQL_DATABASE}', CHAR(96), '.', CHAR(96), table_name, CHAR(96), ' (', GROUP_CONCAT(CONCAT(CHAR(96), column_name, CHAR(96)) ORDER BY ordinal_position SEPARATOR ','), ') SELECT ', GROUP_CONCAT(CONCAT(CHAR(96), column_name, CHAR(96)) ORDER BY ordinal_position SEPARATOR ','), ' FROM ', CHAR(96), 'gcd_dev_import_stage', CHAR(96), '.', CHAR(96), table_name, CHAR(96), ';') FROM information_schema.columns WHERE table_schema = 'gcd_dev_import_stage' AND table_name REGEXP '^(gcd|stddata)_' GROUP BY table_name ORDER BY table_name;")" + catalog_copy_sql="$(mysql_root_query "SET SESSION group_concat_max_len = 1000000; SELECT CONCAT('DELETE FROM ', CHAR(96), '${MYSQL_DATABASE}', CHAR(96), '.', CHAR(96), table_name, CHAR(96), '; INSERT INTO ', CHAR(96), '${MYSQL_DATABASE}', CHAR(96), '.', CHAR(96), table_name, CHAR(96), ' (', GROUP_CONCAT(CONCAT(CHAR(96), column_name, CHAR(96)) ORDER BY ordinal_position SEPARATOR ','), ') SELECT ', GROUP_CONCAT(CONCAT(CHAR(96), column_name, CHAR(96)) ORDER BY ordinal_position SEPARATOR ','), ' FROM ', CHAR(96), 'gcd_dev_import_stage', CHAR(96), '.', CHAR(96), table_name, CHAR(96), ';') FROM information_schema.columns WHERE table_schema = 'gcd_dev_import_stage' AND table_name REGEXP '^(gcd|stddata)_' GROUP BY table_name ORDER BY table_name;")" auto_increment_sql="$(mysql_root_query "SELECT CONCAT('ALTER TABLE ', CHAR(96), '${MYSQL_DATABASE}', CHAR(96), '.', CHAR(96), table_name, CHAR(96), ' AUTO_INCREMENT = 1;') FROM information_schema.columns WHERE table_schema = 'gcd_dev_import_stage' AND table_name REGEXP '^(gcd|stddata)_' AND extra LIKE '%auto_increment%' GROUP BY table_name ORDER BY table_name;")" { printf '%s\n' 'SET FOREIGN_KEY_CHECKS=0;' @@ -255,10 +271,7 @@ copy_dump_catalog() { setup_dump_database() { local archive="$1" - if [[ ! -f "$archive" ]]; then - echo "Dump archive not found: $archive" >&2 - exit 2 - fi + validate_dump_archive "$archive" mysql_root_query 'DROP DATABASE IF EXISTS gcd_dev_import_stage;' echo "Importing the catalog dump into a temporary local staging database..." stream_dump_to_stage "$archive" @@ -275,6 +288,9 @@ seed_docker_development_data() { setup_docker_environment() { local archive="$1" replace="$2" yes="$3" validate_setup_database_name + if [[ -n "$archive" ]]; then + validate_dump_archive "$archive" + fi compose up -d --wait db confirm_setup_replacement "$replace" "$yes" fresh_docker_database