diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 000000000000..0edb8079b01c --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,41 @@ +name: CD + +on: + workflow_dispatch: + +# Restrict GITHUB_TOKEN to the minimum (Scorecard / OpenSSF); deploy uses SSH secrets, not the token. +permissions: + contents: read + +jobs: + deploy: + name: Deploy + runs-on: ubuntu-latest + steps: + - name: Deploy via SSH + uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5 + with: + host: ${{ secrets.SERVER_HOST }} + username: ${{ secrets.SERVER_USER }} + key: ${{ secrets.SERVER_SSH_KEY }} + port: ${{ secrets.SERVER_PORT || 22 }} + script: | + cd /opt/boost-weblate + git pull --depth=1 origin main + git submodule update --init --depth=1 weblate-docker + # Docker build context is .. (repo root); copy submodule ignore to context root + cp weblate-docker/.dockerignore .dockerignore + cd weblate-docker + docker compose down + docker compose up -d --build + + - name: Health check + uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5 + with: + host: ${{ secrets.SERVER_HOST }} + username: ${{ secrets.SERVER_USER }} + key: ${{ secrets.SERVER_SSH_KEY }} + port: ${{ secrets.SERVER_PORT || 22 }} + script: | + sleep 300 + curl -sf http://localhost:8000/healthz/ diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2858b62bda30..2498cbe036b5 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -36,7 +36,7 @@ jobs: with: persist-credentials: false - run: brew update - - run: brew upgrade + # Skip `brew upgrade`: full image upgrades often fail on brew link on GHA macOS; explicit installs below suffice. - run: brew list --versions - run: brew deps --tree --installed - run: brew config diff --git a/.gitignore b/.gitignore index 876575af0ed8..84be477a13cb 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ weblate-*.tar.* *.sublime-* .vscode/* /weblate.egg-info/ +/boost_weblate.egg-info/ /build/ /data/ /data-test/ @@ -41,8 +42,11 @@ weblate-*.tar.* /weblate-openapi.yaml /memray* /mypy.log +/test/ # Local development +/start-weblate.sh +/stop-weblate.sh .cursorignore .cursor/ /logs diff --git a/.gitmodules b/.gitmodules index 47d9fbac5f2e..33f8523a2fe7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,7 @@ [submodule "scripts/spdx-license-list"] path = scripts/spdx-license-list url = https://github.com/spdx/license-list-data.git +[submodule "weblate-docker"] + path = weblate-docker + url = https://github.com/CppDigest/weblate-docker.git + branch = main diff --git a/REUSE.toml b/REUSE.toml index c9bdec47496e..245658d7f907 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -146,3 +146,18 @@ path = "client/yarn.lock" precedence = "aggregate" SPDX-FileCopyrightText = "Michal Čihař " SPDX-License-Identifier = "GPL-3.0-or-later" + +[[annotations]] +path = [ + ".github/workflows/cd.yml", + "docker/**", + "scripts/auto/**", + "scripts/backup/backup_from_server.sh", + "scripts/backup/dump_database.sh", + "scripts/backup/manage_statistics.md", + "scripts/backup/restore_to_local.sh", + "scripts/README_create_project.md" +] +precedence = "aggregate" +SPDX-FileCopyrightText = "Boost Organization " +SPDX-License-Identifier = "GPL-3.0-or-later" diff --git a/codecov.yml b/codecov.yml index f19acfc11f5d..3cbdafcf6b99 100644 --- a/codecov.yml +++ b/codecov.yml @@ -23,6 +23,7 @@ coverage: target: 90 patch: default: + informational: true target: 100 codecov: branch: main diff --git a/docs/specs/openapi.yaml b/docs/specs/openapi.yaml index d73155d6c524..b1a5067cb491 100644 --- a/docs/specs/openapi.yaml +++ b/docs/specs/openapi.yaml @@ -68330,6 +68330,7 @@ components: - po-mono - poxliff - properties + - quickbook - rc - resjson - resourcedictionary @@ -68399,6 +68400,7 @@ components: * `po-mono` - gettext PO file (monolingual) * `poxliff` - XLIFF 1.2 with gettext extensions * `properties` - Java Properties + * `quickbook` - QuickBook file * `rc` - RC file * `resjson` - RESJSON file * `resourcedictionary` - ResourceDictionary file diff --git a/pyproject.toml b/pyproject.toml index 90c8d7a1ee0f..bfeb5db46195 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -295,7 +295,12 @@ ignore = [ "scripts/*", "scripts/*/*", "scripts/*/*/*", - "scripts/*/*/*/*" + "scripts/*/*/*/*", + "weblate-docker", + "weblate-docker/*", + "weblate-docker/*/*", + "weblate-docker/*/*/*", + "weblate-docker/*/*/*/*" ] ignore-bad-ideas = [ "weblate/trans/tests/data/cs.mo" # Test data @@ -607,16 +612,48 @@ max-complexity = 16 # TODO: should be lower "ci/migrate-scripts/*.py" = ["INP001", "S101"] "docs/_ext/djangodocs.py" = ["INP001"] "docs/conf.py" = ["ERA001", "INP001"] -"scripts/*" = ["T201", "T203"] +# Standalone / Django-bootstrap scripts under scripts/auto and scripts/backup (not matched by scripts/*) +"scripts/**/*.py" = [ + "ANN401", + "C901", + "CPY001", + "D205", + "D401", + "E402", + "EXE001", + "FURB118", + "PLC1901", + "PLR0912", + "PLR0914", + "PLR0915", + "PLR0917", + "PLR1702", + "PLR6201", + "RET504", + "RUF005", + "RUF013", + "S110", + "S310", + "SIM102", + "T201", + "T203", + "TRY300" +] "weblate/*/migrations/*.py" = ["RUF012"] "weblate/*/tests**.py" = ["ANN001", "S105", "S106"] "weblate/auth/migrations/0003_fixup_teams.py" = ["T201"] +"weblate/boost_endpoint/services.py" = ["TRY300"] "weblate/examples/*.py" = ["CPY001", "INP001"] +"weblate/formats/asciidoc.py" = ["C901", "PLW1514", "S103"] "weblate/settings_*.py" = ["F405"] "weblate/settings_example.py" = ["ERA001"] +"weblate/trans/autobatchtranslate.py" = ["PLR0917", "RUF059", "TRY300"] "weblate/trans/autofixes/__init__.py" = ["RUF067"] "weblate/trans/models/__init__.py" = ["RUF067"] "weblate/utils/generate_secret_key.py" = ["T201"] +"weblate/utils/openrouter_translator.py" = ["TRY300", "TRY301"] +# Large parser: complexity/Any acceptable until refactor +"weblate/utils/quickbook.py" = ["ANN401", "C901", "PLR0912", "PLR0914", "PLR0915"] [tool.ruff.lint.pylint] # TODO: all these should be lower (or use defaults) @@ -654,12 +691,15 @@ extend-ignore-re = [ [tool.typos.default.extend-identifiers] # TODO: Most of these should be probably fixed, but it requires a database migration ApprovedStringNotificaton = "ApprovedStringNotificaton" +# Boost library names / extensions in generated file lists (not "bitmap" / "make") +bimap = "bimap" ChangedStringNotificaton = "ChangedStringNotificaton" ComponentTranslatedNotificaton = "ComponentTranslatedNotificaton" gir1 = "gir1" # GObject Introspection packages InexistantFiles = "InexistantFiles" LanguageTranslatedNotificaton = "LanguageTranslatedNotificaton" LastAuthorCommentNotificaton = "LastAuthorCommentNotificaton" +mak = "mak" MentionCommentNotificaton = "MentionCommentNotificaton" NewAlertNotificaton = "NewAlertNotificaton" NewAnnouncementNotificaton = "NewAnnouncementNotificaton" @@ -694,6 +734,7 @@ extend-exclude = [ "**.pot", "docs/changes/contributors", "docs/specs", + "scripts/auto/boost-*_libraries_list*.txt", "scripts/codespell.txt", "scripts/spdx-license-list", "weblate/static/js/vendor", diff --git a/scripts/auto/create_component_and_add_translation.py b/scripts/auto/create_component_and_add_translation.py index df6a0a7d776e..6e91e235c76e 100755 --- a/scripts/auto/create_component_and_add_translation.py +++ b/scripts/auto/create_component_and_add_translation.py @@ -232,9 +232,9 @@ def create_component_wrapper(config: dict[str, Any]) -> tuple[str, str]: if not creator.check_connection(): msg = "Failed to connect to Weblate API" - raise Exception(msg) + raise RuntimeError(msg) - project, component, project_slug, component_slug = setup_project_and_component( + _project, component, project_slug, component_slug = setup_project_and_component( creator, config ) @@ -249,7 +249,7 @@ def create_component_wrapper(config: dict[str, Any]) -> tuple[str, str]: # Verify component is accessible if not _verify_component_accessible(creator, project_slug, component_slug): msg = "Component not accessible after creation" - raise Exception(msg) + raise RuntimeError(msg) print("\n[SUCCESS] Component created and ready!", flush=True) print(f"[INFO] URL: {component['web_url']}", flush=True) diff --git a/scripts/auto/setup_project.py b/scripts/auto/setup_project.py index 4a84ffa6b235..38cb85372870 100644 --- a/scripts/auto/setup_project.py +++ b/scripts/auto/setup_project.py @@ -48,7 +48,9 @@ def clone_repository(repo_url: str, branch: str, target_dir: str) -> bool: # Use repo URL as-is so SSH keys work when repo_url is git@github.com:... clone_url = repo_url cmd = ["git", "clone", "-b", branch, "--depth", "1", clone_url, target_dir] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=300) + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=300, check=False + ) if result.returncode != 0: print(f"[ERROR] Failed to clone repository: {result.stderr}") @@ -389,6 +391,7 @@ def create_components_from_setup_files( capture_output=True, text=True, timeout=300, # 5 minute timeout per component + check=False, ) elapsed = time.time() - start_time @@ -423,6 +426,7 @@ def create_components_from_setup_files( capture_output=True, text=True, timeout=300, + check=False, ) if retry.returncode == 0: elapsed2 = time.time() - start_time diff --git a/scripts/backup/backup_from_server.sh b/scripts/backup/backup_from_server.sh index d948c0e70496..5da1b06042f3 100755 --- a/scripts/backup/backup_from_server.sh +++ b/scripts/backup/backup_from_server.sh @@ -37,13 +37,11 @@ export PGPASSWORD="$DB_PASSWORD" # Use pg_dump plain SQL format for compatibility with psql restores DB_DUMP_FILE="weblate_database_$TIMESTAMP.sql" -pg_dump -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" \ +if pg_dump -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" \ --format=plain \ --no-owner \ --no-privileges \ - -f "$DB_DUMP_FILE" - -if [ $? -eq 0 ]; then + -f "$DB_DUMP_FILE"; then echo -e "${GREEN}✓ Database backup created: $DB_DUMP_FILE${NC}" ls -lh "$DB_DUMP_FILE" else diff --git a/scripts/backup/dump_database.sh b/scripts/backup/dump_database.sh index 3fa6d3b53bdf..df969c2438d2 100755 --- a/scripts/backup/dump_database.sh +++ b/scripts/backup/dump_database.sh @@ -8,9 +8,11 @@ OUTPUT_FILE="$HOME/boost-weblate/weblate_backup_$(date +%Y%m%d_%H%M%S).sql" export PGPASSWORD="weblate" -echo "-- Database dump with ordered tables and rows" > "$OUTPUT_FILE" -echo "-- Generated: DUMMY_TIMESTAMP" >> "$OUTPUT_FILE" -echo "" >> "$OUTPUT_FILE" +{ + echo "-- Database dump with ordered tables and rows" + echo "-- Generated: DUMMY_TIMESTAMP" + echo "" +} > "$OUTPUT_FILE" # Dump schema only (without data) echo "Dumping schema..." @@ -21,11 +23,13 @@ pg_dump -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" \ sed 's/\\restrict [^[:space:]]*/\\restrict DUMMY_TOKEN/g' | sed 's/\\unrestrict [^[:space:]]*/\\unrestrict DUMMY_TOKEN/g' >> "$OUTPUT_FILE" -echo "" >> "$OUTPUT_FILE" -echo "-- Data dump with ordered rows" >> "$OUTPUT_FILE" -echo "-- Disable foreign key checks during data load" >> "$OUTPUT_FILE" -echo "SET session_replication_role = 'replica';" >> "$OUTPUT_FILE" -echo "" >> "$OUTPUT_FILE" +{ + echo "" + echo "-- Data dump with ordered rows" + echo "-- Disable foreign key checks during data load" + echo "SET session_replication_role = 'replica';" + echo "" +} >> "$OUTPUT_FILE" # Get all tables with their primary key columns, ordered by foreign key dependencies # Use pg_dump's internal dependency ordering by extracting table order from a test dump @@ -72,10 +76,9 @@ ORDER BY fk_count, t.tablename; ") # Dump data for each table, ordered by primary key -while IFS='|' read -r tablename pk_column fk_count; do +while IFS='|' read -r tablename pk_column _fk_count; do tablename=$(echo "$tablename" | xargs) pk_column=$(echo "$pk_column" | xargs) - # fk_count is ignored but needed to read all columns if [ -z "$tablename" ]; then continue @@ -87,8 +90,10 @@ while IFS='|' read -r tablename pk_column fk_count; do row_count=$(psql -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" -t -c "SELECT COUNT(*) FROM \"$tablename\";" | xargs) if [ "$row_count" -gt 0 ]; then - echo "" >> "$OUTPUT_FILE" - echo "-- Data for table: $tablename (ordered by $pk_column)" >> "$OUTPUT_FILE" + { + echo "" + echo "-- Data for table: $tablename (ordered by $pk_column)" + } >> "$OUTPUT_FILE" # Get column names for the table columns=$(psql -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" -t -c " @@ -107,8 +112,10 @@ while IFS='|' read -r tablename pk_column fk_count; do : # Success - data written else # Fallback: dump without ordering if ORDER BY fails - echo "-- Warning: Could not order by $pk_column, dumping without order" >> "$OUTPUT_FILE" - psql -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" -c "COPY \"$tablename\" TO STDOUT;" >> "$OUTPUT_FILE" 2> /dev/null + { + echo "-- Warning: Could not order by $pk_column, dumping without order" + psql -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" -c "COPY \"$tablename\" TO STDOUT;" + } >> "$OUTPUT_FILE" 2> /dev/null fi # End COPY block @@ -116,11 +123,13 @@ while IFS='|' read -r tablename pk_column fk_count; do fi done <<< "$TABLES" -echo "" >> "$OUTPUT_FILE" -echo "-- Re-enable foreign key checks" >> "$OUTPUT_FILE" -echo "SET session_replication_role = 'origin';" >> "$OUTPUT_FILE" -echo "" >> "$OUTPUT_FILE" -echo "-- End of dump" >> "$OUTPUT_FILE" +{ + echo "" + echo "-- Re-enable foreign key checks" + echo "SET session_replication_role = 'origin';" + echo "" + echo "-- End of dump" +} >> "$OUTPUT_FILE" echo "Database dump completed: $OUTPUT_FILE" ls -lh "$OUTPUT_FILE" diff --git a/scripts/backup/recalculate_stats.py b/scripts/backup/recalculate_stats.py index d5dc966708d5..b90ef2566835 100644 --- a/scripts/backup/recalculate_stats.py +++ b/scripts/backup/recalculate_stats.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# Copyright © Boost Organization +# +# SPDX-License-Identifier: GPL-3.0-or-later + """Recalculate statistics for all components in a project.""" import os @@ -11,7 +15,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "weblate.settings") django.setup() -from weblate.trans.models import Project +from weblate.trans.models import Project # pylint: disable=wrong-import-position def recalculate_stats(project_slug): diff --git a/scripts/backup/restore_to_local.sh b/scripts/backup/restore_to_local.sh index dfed797d4990..80ab1c120cde 100755 --- a/scripts/backup/restore_to_local.sh +++ b/scripts/backup/restore_to_local.sh @@ -42,9 +42,9 @@ fi cd "$BACKUP_DIR" -# Find backup files -DB_SQL=$(ls -1 weblate_database_*.sql 2> /dev/null | head -1) -FILES_ARCHIVE=$(ls -1 weblate_files_*.tar.gz 2> /dev/null | head -1) +# Find backup files (find avoids SC2012 issues with ls + globs) +DB_SQL=$(find . -maxdepth 1 -name 'weblate_database_*.sql' 2> /dev/null | head -n1 | sed 's|^\./||') +FILES_ARCHIVE=$(find . -maxdepth 1 -name 'weblate_files_*.tar.gz' 2> /dev/null | head -n1 | sed 's|^\./||') if [ -z "$DB_SQL" ] && [ -z "$FILES_ARCHIVE" ]; then echo -e "${RED}Error: No backup files found in '$BACKUP_DIR'${NC}" @@ -64,7 +64,7 @@ echo -e "DATA_DIR: ${YELLOW}$DATA_DIR${NC}" echo "" # Confirm before proceeding -read -p "This will OVERWRITE your local Weblate database and files. Continue? (yes/no): " confirm +read -r -p "This will OVERWRITE your local Weblate database and files. Continue? (yes/no): " confirm if [ "$confirm" != "yes" ]; then echo -e "${YELLOW}Restore cancelled${NC}" exit 0 @@ -84,11 +84,9 @@ if [ -n "$DB_SQL" ]; then createdb -h "$DB_HOST" -U "$DB_USER" "$DB_NAME" echo -e "${YELLOW}Restoring database from $DB_SQL...${NC}" - psql -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" \ + if psql -h "$DB_HOST" -U "$DB_USER" -d "$DB_NAME" \ -v ON_ERROR_STOP=1 \ - -f "$DB_SQL" - - if [ $? -eq 0 ]; then + -f "$DB_SQL"; then echo -e "${GREEN}✓ Database restored successfully${NC}" else echo -e "${RED}✗ Database restore failed!${NC}" @@ -112,6 +110,7 @@ fi if [ -n "$WEBLATE_DIR" ] && [ -f "$WEBLATE_DIR/manage.py" ]; then cd "$WEBLATE_DIR" + # shellcheck disable=SC1091 source weblate-env/bin/activate 2> /dev/null || true python manage.py shell << 'PYTHON_EOF' @@ -166,9 +165,7 @@ if [ -n "$FILES_ARCHIVE" ]; then mkdir -p "$DATA_DIR" echo -e "${YELLOW}Extracting files to $DATA_DIR...${NC}" - tar -xzf "$FILES_ARCHIVE" -C "$DATA_DIR" - - if [ $? -eq 0 ]; then + if tar -xzf "$FILES_ARCHIVE" -C "$DATA_DIR"; then echo -e "${GREEN}✓ Files restored successfully${NC}" # Set proper permissions (adjust user/group as needed) @@ -188,6 +185,7 @@ if [ -n "$WEBLATE_DIR" ] && [ -f "$WEBLATE_DIR/manage.py" ]; then echo -e "${GREEN}[4/4] Running post-restore steps...${NC}" cd "$WEBLATE_DIR" + # shellcheck disable=SC1091 source weblate-env/bin/activate 2> /dev/null || true echo -e "${YELLOW}Updating Git repositories...${NC}" diff --git a/scripts/backup/sync_database_to_files.py b/scripts/backup/sync_database_to_files.py index 16d2ee170829..dd7a49f58340 100755 --- a/scripts/backup/sync_database_to_files.py +++ b/scripts/backup/sync_database_to_files.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# Copyright © Boost Organization +# +# SPDX-License-Identifier: GPL-3.0-or-later + """ Script to synchronize Weblate database translations to files. @@ -21,9 +25,12 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "weblate.settings") django.setup() -from django.db import transaction +from django.db import transaction # pylint: disable=wrong-import-position -from weblate.trans.models import Component, Project +from weblate.trans.models import ( # pylint: disable=wrong-import-position + Component, + Project, +) def sync_component(component: Component) -> bool: diff --git a/scripts/backup/update_push_urls.py b/scripts/backup/update_push_urls.py index 4a8465c0a85a..bf8624cb253d 100755 --- a/scripts/backup/update_push_urls.py +++ b/scripts/backup/update_push_urls.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# Copyright © Boost Organization +# +# SPDX-License-Identifier: GPL-3.0-or-later + """ Script to update repository URLs (both repo and push) and push branch for all Weblate components. @@ -45,7 +49,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "weblate.settings") django.setup() -from weblate.trans.models import Component +from weblate.trans.models import Component # pylint: disable=wrong-import-position def update_push_urls( diff --git a/scripts/spdx-license-list b/scripts/spdx-license-list index c4a7237ec8f4..f7b69b12cf4c 160000 --- a/scripts/spdx-license-list +++ b/scripts/spdx-license-list @@ -1 +1 @@ -Subproject commit c4a7237ec8f4654e867546f9f409749300f1bf4c +Subproject commit f7b69b12cf4c063d9c42c0c72945978c87f3192c diff --git a/start-weblate.sh b/start-weblate.sh deleted file mode 100755 index 4e1973380769..000000000000 --- a/start-weblate.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -# Weblate Startup Script -# One-click operation to start Weblate server and Celery workers - -# Colors for output -GREEN='\033[0;32m' -BLUE='\033[0;34m' -RED='\033[0;31m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -echo -e "${BLUE}========================================${NC}" -echo -e "${BLUE} Weblate Startup Script${NC}" -echo -e "${BLUE}========================================${NC}" -echo "" - -# Set log directory (use DATA_DIR/logs if DATA_DIR is set, otherwise use project logs directory) -if [ -z "$DATA_DIR" ]; then - LOG_DIR="$HOME/boost-weblate/logs" -else - LOG_DIR="$DATA_DIR/logs" -fi - -# Create log directory if it doesn't exist -mkdir -p "$LOG_DIR" - -# Navigate to Weblate directory (IMPORTANT: Must be here before starting Celery) -cd $HOME/boost-weblate - -# Activate virtual environment -echo -e "${YELLOW}Activating virtual environment...${NC}" -source $HOME/boost-weblate/weblate-env/bin/activate - -# Check if Celery is already running -if pgrep -f "celery.*weblate" > /dev/null; then - echo -e "${YELLOW}Celery workers are already running!${NC}" -else - echo -e "${GREEN}Starting Celery workers...${NC}" - # Start Celery with custom log directory - export CELERY_APP=weblate.utils - python -m celery multi start celery \ - "--pidfile=$LOG_DIR/weblate-%n.pid" \ - "--logfile=$LOG_DIR/weblate-%n%I.log" \ - --loglevel=DEBUG \ - --queues:celery=celery,notify,memory,translate,backup \ - --beat:celery - sleep 3 - echo -e "${GREEN}✓ Celery workers started${NC}" -fi - -# Check if server is already running -if pgrep -f "weblate runserver" > /dev/null; then - echo -e "${YELLOW}Django server is already running!${NC}" -else - echo -e "${GREEN}Starting Django development server...${NC}" - nohup weblate runserver > "$LOG_DIR/server.log" 2>&1 & - sleep 2 - echo -e "${GREEN}✓ Django server started on http://localhost:8000${NC}" -fi - -echo "" -echo -e "${BLUE}========================================${NC}" -echo -e "${GREEN}✓ Weblate is now running!${NC}" -echo "" -echo -e " Access Weblate at: ${BLUE}http://localhost:8000${NC}" -echo "" -echo -e " Logs directory: ${BLUE}$LOG_DIR${NC}" -echo -e " Logs:" -echo -e " - Server: ${YELLOW}tail -f $LOG_DIR/server.log${NC}" -echo -e " - Celery: ${YELLOW}tail -f $LOG_DIR/weblate-*.log${NC}" -echo "" -echo -e " To stop Weblate, run: ${YELLOW}./stop-weblate.sh${NC}" -echo -e "${BLUE}========================================${NC}" diff --git a/stop-weblate.sh b/stop-weblate.sh deleted file mode 100755 index da62648ff8f0..000000000000 --- a/stop-weblate.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# Weblate Stop Script -# Stop all Weblate server and Celery workers - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -echo -e "${RED}========================================${NC}" -echo -e "${RED} Stopping Weblate${NC}" -echo -e "${RED}========================================${NC}" -echo "" - -# Stop Django server -if pgrep -f "weblate runserver" > /dev/null; then - echo -e "${YELLOW}Stopping Django server...${NC}" - pkill -f "weblate runserver" - sleep 1 - echo -e "${GREEN}✓ Django server stopped${NC}" -else - echo -e "${YELLOW}Django server is not running${NC}" -fi - -# Stop Celery workers -if pgrep -f "celery.*weblate" > /dev/null; then - echo -e "${YELLOW}Stopping Celery workers...${NC}" - pkill -f "celery.*weblate" - sleep 1 - echo -e "${GREEN}✓ Celery workers stopped${NC}" -else - echo -e "${YELLOW}Celery workers are not running${NC}" -fi - -echo "" -echo -e "${GREEN}✓ Weblate has been stopped${NC}" -echo -e "${RED}========================================${NC}" diff --git a/uv.lock b/uv.lock index db3d0cfd9342..cd06bb2cfbcd 100644 --- a/uv.lock +++ b/uv.lock @@ -443,6 +443,11 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/84/85/57c314a6b35336efbbdc13e5fc9ae13f6b60a0647cfa7c1221178ac6d8ae/brotlicffi-1.2.0.0.tar.gz", hash = "sha256:34345d8d1f9d534fcac2249e57a4c3c8801a33c9942ff9f8574f67a175e17adb", size = 476682, upload-time = "2025-11-21T18:17:57.334Z" } wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/87/ba6298c3d7f8d66ce80d7a487f2a487ebae74a79c6049c7c2990178ce529/brotlicffi-1.2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b13fb476a96f02e477a506423cb5e7bc21e0e3ac4c060c20ba31c44056e38c68", size = 433038, upload-time = "2026-03-05T17:57:37.96Z" }, + { url = "https://files.pythonhosted.org/packages/00/49/16c7a77d1cae0519953ef0389a11a9c2e2e62e87d04f8e7afbae40124255/brotlicffi-1.2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17db36fb581f7b951635cd6849553a95c6f2f53c1a707817d06eae5aeff5f6af", size = 1541124, upload-time = "2026-03-05T17:57:39.488Z" }, + { url = "https://files.pythonhosted.org/packages/e8/17/fab2c36ea820e2288f8c1bf562de1b6cd9f30e28d66f1ce2929a4baff6de/brotlicffi-1.2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40190192790489a7b054312163d0ce82b07d1b6e706251036898ce1684ef12e9", size = 1541983, upload-time = "2026-03-05T17:57:41.061Z" }, + { url = "https://files.pythonhosted.org/packages/78/c9/849a669b3b3bb8ac96005cdef04df4db658c33443a7fc704a6d4a2f07a56/brotlicffi-1.2.0.0-cp314-cp314t-win32.whl", hash = "sha256:a8079e8ecc32ecef728036a1d9b7105991ce6a5385cf51ee8c02297c90fb08c2", size = 349046, upload-time = "2026-03-05T17:57:42.76Z" }, + { url = "https://files.pythonhosted.org/packages/a4/25/09c0fd21cfc451fa38ad538f4d18d8be566746531f7f27143f63f8c45a9f/brotlicffi-1.2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:ca90c4266704ca0a94de8f101b4ec029624273380574e4cf19301acfa46c61a0", size = 385653, upload-time = "2026-03-05T17:57:44.224Z" }, { url = "https://files.pythonhosted.org/packages/e4/df/a72b284d8c7bef0ed5756b41c2eb7d0219a1dd6ac6762f1c7bdbc31ef3af/brotlicffi-1.2.0.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:9458d08a7ccde8e3c0afedbf2c70a8263227a68dea5ab13590593f4c0a4fd5f4", size = 432340, upload-time = "2025-11-21T18:17:42.277Z" }, { url = "https://files.pythonhosted.org/packages/74/2b/cc55a2d1d6fb4f5d458fba44a3d3f91fb4320aa14145799fd3a996af0686/brotlicffi-1.2.0.0-cp38-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:84e3d0020cf1bd8b8131f4a07819edee9f283721566fe044a20ec792ca8fd8b7", size = 1534002, upload-time = "2025-11-21T18:17:43.746Z" }, { url = "https://files.pythonhosted.org/packages/e4/9c/d51486bf366fc7d6735f0e46b5b96ca58dc005b250263525a1eea3cd5d21/brotlicffi-1.2.0.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:33cfb408d0cff64cd50bef268c0fed397c46fbb53944aa37264148614a62e990", size = 1536547, upload-time = "2025-11-21T18:17:45.729Z" }, diff --git a/weblate-docker b/weblate-docker new file mode 160000 index 000000000000..9ba5d7396580 --- /dev/null +++ b/weblate-docker @@ -0,0 +1 @@ +Subproject commit 9ba5d7396580ccda8bd4e34f109e59831d60f5be diff --git a/weblate/boost_endpoint/services.py b/weblate/boost_endpoint/services.py index 5c0ea32500f7..fcdef71e9759 100644 --- a/weblate/boost_endpoint/services.py +++ b/weblate/boost_endpoint/services.py @@ -26,7 +26,7 @@ import tempfile import time from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any, cast from django.conf import settings from django.contrib.messages import get_messages @@ -39,6 +39,9 @@ from weblate.utils.errors import report_error from weblate.vcs.base import RepositoryError +if TYPE_CHECKING: + from weblate.lang.models import LanguageQuerySet + # Weblate API limit for component name and slug (Component.name / Component.slug max_length) MAX_COMPONENT_NAME_LENGTH = 100 MAX_COMPONENT_SLUG_LENGTH = 100 @@ -112,6 +115,7 @@ def get_extension_to_format(self) -> dict[str, str]: def get_supported_extensions(self) -> set[str]: """ Set of supported file extensions (from Weblate formats). + If self.extensions is non-empty, restrict to those that are both Weblate-supported and in the list. """ @@ -135,7 +139,13 @@ def clone_repository(self, submodule: str, target_dir: str, branch: str) -> bool try: LOGGER.info("Cloning %s to %s", repo_url, target_dir) cmd = ["git", "clone", "-b", branch, "--depth", "1", repo_url, target_dir] - result = subprocess.run(cmd, capture_output=True, text=True, timeout=300) + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=300, + check=False, + ) if result.returncode != 0: LOGGER.error("Failed to clone: %s", result.stderr) @@ -155,6 +165,7 @@ def clone_repository(self, submodule: str, target_dir: str, branch: str) -> bool def scan_documentation_files(self, repo_dir: str) -> list[dict[str, Any]]: """ Scan repo for doc files; return list of in-memory component configs. + Only files in subfolders are included; files in repo root are skipped. Uses get_supported_extensions() which respects self.extensions when set. """ @@ -206,7 +217,7 @@ def generate_component_config( dir_path = path_obj.parent # Generate component name from path (include extension so doc/intro.adoc vs doc/intro.md differ) - component_name_parts = [] + component_name_parts: list[str] = [] if str(dir_path) != ".": component_name_parts.extend(dir_path.parts) component_name_parts.append(filename_base) @@ -418,6 +429,7 @@ def create_or_update_component( def _do_update_git_only(self, component: Component, request) -> bool: """ Perform only the git update (fetch, merge/rebase). Does not call create_translations. + Mirrors Component.do_update lock block + push_if_needed; caller must call create_translations_immediate after. """ @@ -558,7 +570,9 @@ def add_language_to_component(self, component: Component, request=None) -> bool: # (2) get_all_available_languages() + add_more filter: DB only. Ensure lang_code is in the # allowed set (not already in component; if user lacks add_more, restrict to basic/project # languages). Fail fast before any I/O so we do not sync when language is not addable. - base_languages = component.get_all_available_languages() + base_languages = cast( + "LanguageQuerySet", component.get_all_available_languages() + ) if not request.user.has_perm("translation.add_more", component): base_languages = base_languages.filter_for_add(component.project) if not base_languages.filter(pk=language.pk).exists(): @@ -635,8 +649,15 @@ def _delete_component_and_commit_removal( name = component.name base_path = component.full_path repo_owner = component.linked_component if component.is_repo_link else component - push_branch = repo_owner.push_branch - push_url = repo_owner.push + if repo_owner is None: + LOGGER.warning( + "Cannot push after delete: no linked component for %s", component.slug + ) + push_branch = None + push_url = None + else: + push_branch = repo_owner.push_branch + push_url = repo_owner.push translation_files = [ os.path.join(base_path, t.filename) for t in component.translation_set.exclude( @@ -665,7 +686,7 @@ def _delete_component_and_commit_removal( # Stage only the removed files (not all tracked changes) rel_paths = [os.path.relpath(p, base_path) for p in actually_removed] subprocess.run( - ["git", "-C", base_path, "add", "--"] + rel_paths, + ["git", "-C", base_path, "add", "--", *rel_paths], check=True, capture_output=True, timeout=60, @@ -675,6 +696,7 @@ def _delete_component_and_commit_removal( capture_output=True, text=True, timeout=10, + check=False, ) if status.stdout.strip(): author = ( @@ -730,7 +752,7 @@ def process_submodule( if self.temp_dir is None: msg = "process_submodule requires temp_dir; call process_all() instead" raise TypeError(msg) - result = { + result: dict[str, Any] = { "submodule": submodule, "success": False, "components_created": 0, @@ -827,7 +849,7 @@ def process_all( self.temp_dir = tempfile.mkdtemp(prefix="boost_endpoint_") LOGGER.info("Using temp directory: %s", self.temp_dir) - results = { + results: dict[str, Any] = { "total_submodules": len(submodules), "successful": 0, "failed": 0, diff --git a/weblate/boost_endpoint/views.py b/weblate/boost_endpoint/views.py index c2eaf87ee4a6..998ec8499e47 100644 --- a/weblate/boost_endpoint/views.py +++ b/weblate/boost_endpoint/views.py @@ -18,7 +18,7 @@ class BoostEndpointInfo(APIView): permission_classes = (IsAuthenticated,) - def get(self, request, format=None): + def get(self, request, format=None): # pylint: disable=redefined-builtin # noqa: A002 """Return Boost endpoint module info.""" return Response( { @@ -33,7 +33,7 @@ class AddOrUpdateView(APIView): permission_classes = (IsAuthenticated,) - def post(self, request, format=None): + def post(self, request, format=None): # pylint: disable=redefined-builtin # noqa: A002 """ Create or update Boost documentation components. diff --git a/weblate/formats/asciidoc.py b/weblate/formats/asciidoc.py old mode 100755 new mode 100644 index 79292639fc1c..fc9c2e4c7646 --- a/weblate/formats/asciidoc.py +++ b/weblate/formats/asciidoc.py @@ -1,9 +1,8 @@ -""" -AsciiDoc file format support for Weblate. +# Copyright © Boost Organization +# +# SPDX-License-Identifier: GPL-3.0-or-later -This format handles .adoc files for documentation translation using po4a. -Based on po4a's AsciiDoc module for extraction and translation. -""" +"""AsciiDoc file format support for Weblate (po4a-based).""" import os import pathlib @@ -93,10 +92,12 @@ def convertfile(self, storefile, template_store): # Get storefile path (localized file for po4a-gettextize) # If storefile is a file object without a name, we need to create a temp file + storefile_path: str | None if isinstance(storefile, str): storefile_path = storefile else: - storefile_path = getattr(storefile, "name", None) + raw_name = getattr(storefile, "name", None) + storefile_path = raw_name if isinstance(raw_name, str) else None # When template_store is None (e.g., during base file validation), # use storefile as both template and localized file @@ -267,6 +268,10 @@ def save_content(self, handle) -> None: Uses po4a-translate to merge PO translations back into AsciiDoc template. """ + if self.template_store is None: + msg = "AsciiDoc: cannot save without template store" + report_error(msg) + raise RuntimeError(msg) # Get template AsciiDoc file path template_path = self.template_store.storefile if hasattr(template_path, "name"): @@ -289,7 +294,7 @@ def save_content(self, handle) -> None: # Use msgattrib to clear fuzzy flags from the PO file # This allows po4a-translate to use fuzzy translations try: - result = subprocess.run( + msgattrib_result = subprocess.run( [ "msgattrib", "--clear-fuzzy", @@ -299,9 +304,9 @@ def save_content(self, handle) -> None: text=False, # Capture as bytes to preserve encoding check=False, ) - if result.returncode == 0 and result.stdout: + if msgattrib_result.returncode == 0 and msgattrib_result.stdout: # Write the output to the second temporary file - pathlib.Path(tmp_po_path_02).write_bytes(result.stdout) + pathlib.Path(tmp_po_path_02).write_bytes(msgattrib_result.stdout) else: # If msgattrib fails, use the original PO file tmp_po_path_02 = tmp_po_path_01 @@ -324,7 +329,7 @@ def save_content(self, handle) -> None: msgfmt_wrapper_path = os.path.join(tmp_bin_dir, "msgfmt") # Create wrapper script that always succeeds - with open(msgfmt_wrapper_path, "w") as wrapper: + with open(msgfmt_wrapper_path, "w", encoding="utf-8") as wrapper: wrapper.write("#!/bin/bash\n") wrapper.write( "# Wrapper to bypass msgfmt validation - always succeed to allow po4a-translate to proceed\n" @@ -349,7 +354,7 @@ def save_content(self, handle) -> None: # -m: template file (master) # -p: PO file with translations # -l: output translated AsciiDoc file - result = subprocess.run( + po4a_result = subprocess.run( [ "po4a-translate", "-f", @@ -390,9 +395,10 @@ def save_content(self, handle) -> None: handle.write(content.encode("utf-8")) else: # Translation failed: raise exception to prevent silent failure + stderr_text = po4a_result.stderr or "" error_msg = ( - f"po4a-translate failed: {result.stderr}" - if result.returncode != 0 + f"po4a-translate failed: {stderr_text}" + if po4a_result.returncode != 0 else "po4a-translate failed: no output file generated" ) report_error(error_msg) @@ -400,8 +406,9 @@ def save_content(self, handle) -> None: raise RuntimeError(error_msg) # Report warnings if any (but don't fail on warnings) - if result.returncode != 0 and result.stderr: - report_error(f"po4a-translate warning: {result.stderr}") + if po4a_result.returncode != 0 and po4a_result.stderr: + warn_err = po4a_result.stderr + report_error(f"po4a-translate warning: {warn_err}") except subprocess.CalledProcessError as e: error_msg = f"po4a-translate error: {e.stderr}" report_error(error_msg) diff --git a/weblate/formats/quickbook.py b/weblate/formats/quickbook.py index bbe302c1497d..a60832212ded 100644 --- a/weblate/formats/quickbook.py +++ b/weblate/formats/quickbook.py @@ -13,7 +13,7 @@ from __future__ import annotations from pathlib import Path -from typing import TYPE_CHECKING, BinaryIO +from typing import IO, TYPE_CHECKING from django.utils.translation import gettext_lazy from translate.storage.pypo import pofile @@ -47,7 +47,7 @@ class QuickBookFormat(ConvertFormat): def convertfile( self, - storefile: str | BinaryIO, + storefile: IO[bytes], template_store: TranslationFormat | None, ) -> TranslationStore: """Extract translatable strings from a .qbk file, returning a ``pofile``.""" @@ -61,11 +61,8 @@ def convertfile( template_path = tf if template_path is None: - # Fall back: use storefile itself as the template. - if isinstance(storefile, str): - template_path = storefile - else: - template_path = getattr(storefile, "name", None) + # Fall back: use storefile path as the template. + template_path = getattr(storefile, "name", None) if template_path is None: report_error("QuickBook: cannot determine template file path") @@ -88,24 +85,48 @@ def convertfile( filename = Path(template_path).name store = qbk_to_po(content, filename, self.existing_units) - # When loading the source-language file (storefile IS the template), set - # target = source on every unit. This mirrors what po4a-gettextize produces - # when given the same file for both master and localized, and is required so - # that Weblate stores the correct (non-empty) translation for the source - # language in a monolingual component. - storefile_path = ( - getattr(storefile, "name", storefile) - if not isinstance(storefile, str) - else storefile - ) + storefile_path: str | None = getattr(storefile, "name", None) if storefile_path == template_path: + # Loading the source-language file: set target = source on every unit + # so Weblate stores a non-empty translation for the source language. for unit in store.units: if not unit.isheader(): unit.target = unit.source + # Loading a translated .qbk file: parse it and pair its segments + # positionally with the template segments to populate msgstr values. + # This mirrors what po4a-gettextize does when given both -m and -l. + elif storefile_path is None: + report_error( + "QuickBook: cannot load translated .qbk without a filesystem path" + ) + else: + try: + translated_content = Path(storefile_path).read_text(encoding="utf-8") + translated_store = qbk_to_po( + translated_content, Path(storefile_path).name + ) + trans_units = [u for u in translated_store.units if not u.isheader()] + tmpl_units = [u for u in store.units if not u.isheader()] + if len(tmpl_units) != len(trans_units): + report_error( + "QuickBook: refusing positional import: segment count mismatch " + f"(file={storefile_path!s}, name={Path(storefile_path).name!s}, " + f"template_units={len(tmpl_units)}, translated_units={len(trans_units)})" + ) + else: + for tmpl_unit, trans_unit in zip( + tmpl_units, trans_units, strict=True + ): + if trans_unit.source: + tmpl_unit.target = trans_unit.source + except Exception as exc: + report_error( + f"QuickBook: cannot read translated file {storefile_path}: {exc}" + ) return store - def save_content(self, handle: BinaryIO) -> None: + def save_content(self, handle: IO[bytes]) -> None: """Write the translated .qbk by applying PO translations to the template.""" template_store = getattr(self, "template_store", None) if template_store is None: diff --git a/weblate/settings_docker.py b/weblate/settings_docker.py index 9e0812341f17..2bb6daee7c2a 100644 --- a/weblate/settings_docker.py +++ b/weblate/settings_docker.py @@ -772,6 +772,7 @@ "customize", # Weblate apps on top to override Django locales and templates "weblate.addons", + "weblate.boost_endpoint", "weblate.auth", "weblate.checks", "weblate_fonts", @@ -1493,6 +1494,14 @@ SENTRY_SEND_PII = get_env_bool("SENTRY_SEND_PII", False) ZAMMAD_URL = get_env_str("WEBLATE_ZAMMAD_URL") +# boost-weblate specific settings +AUTO_BATCH_TRANSLATE_VIA_OPENROUTER = get_env_bool( + "AUTO_BATCH_TRANSLATE_VIA_OPENROUTER", True +) +BOOST_ENDPOINT_ADD_TRANSLATION_SECONDS = get_env_int( + "BOOST_ENDPOINT_ADD_TRANSLATION_SECONDS", 300 +) + ADDITIONAL_CONFIG = Path("/app/data/settings-override.py") if ADDITIONAL_CONFIG.exists(): code = compile( diff --git a/weblate/trans/autobatchtranslate.py b/weblate/trans/autobatchtranslate.py index ad9584e036dc..10752b09ffa1 100644 --- a/weblate/trans/autobatchtranslate.py +++ b/weblate/trans/autobatchtranslate.py @@ -1,4 +1,4 @@ -# Copyright © William +# Copyright © Boost Organization # # SPDX-License-Identifier: GPL-3.0-or-later @@ -13,7 +13,7 @@ def auto_translate_via_openrouter(translation: Translation) -> Translation: """Auto translation via OpenRouter (modularized).""" # 1) Resolve configuration - api_key, model, config_source = _resolve_openrouter_config(translation) + api_key, model, _config_source = _resolve_openrouter_config(translation) if not api_key or not model: translation.log_warning( "OpenRouter configuration not found, skipping auto-translation for: %s", @@ -106,11 +106,6 @@ def _resolve_openrouter_config(translation: Translation): def _prepare_batch_request(translation: Translation): - - from weblate.utils.openrouter_translator import ( - OpenRouterTranslator, # noqa: F401 (import side-effect for types) - ) - # Collect untranslated units units_qs = translation.unit_set.all().order_by("position") if not units_qs.exists(): diff --git a/weblate/trans/models/component.py b/weblate/trans/models/component.py index 6d32cfeb976c..e043e9d2972f 100644 --- a/weblate/trans/models/component.py +++ b/weblate/trans/models/component.py @@ -8,6 +8,7 @@ import re import time from collections import defaultdict +from datetime import UTC from glob import glob from itertools import chain from typing import TYPE_CHECKING, Any, TypedDict, cast @@ -29,6 +30,7 @@ from django.db.models import Count, F, Q from django.db.models.signals import m2m_changed from django.dispatch import receiver +from django.utils import timezone from django.utils.functional import cached_property from django.utils.html import format_html, format_html_join from django.utils.safestring import mark_safe @@ -2917,7 +2919,6 @@ def _create_translations( # noqa: C901,PLR0915 request=request, change=change, ) - # transaction.on_commit(lambda: self.auto_translate_via_openrouter()) except InvalidTemplateError as error: self.log_warning( "skipping update due to error in parsing template: %s", @@ -4065,8 +4066,6 @@ def fail_message(message: StrOrPromise) -> None: "language_code": code, }, ) - # if create_translations: - # translation = translation.auto_translate_via_openrouter() # Make it clear that there is no change for the newly created translation # to avoid expensive last change lookup in stats while committing changes. if created: @@ -4394,6 +4393,7 @@ def autobatchtranslate_via_openrouter( user_id=request.user.id if request is not None else None, file_sync=file_sync, ) + return None def _autobatchtranslate_via_openrouter_immediate( self, @@ -4457,8 +4457,6 @@ def _autobatchtranslate_via_openrouter_immediate( except Exception as e: # Handle IntegrityError and other exceptions gracefully # This can happen if another task already created the units - from django.db import IntegrityError - if not isinstance(e, IntegrityError): self.log_error( "Autobatch translation: error during file parsing for language %s in component %s (ID: %d): %s", @@ -4526,8 +4524,6 @@ def _autobatchtranslate_via_openrouter_immediate( # This ensures file sync runs after all translations are processed # Only schedule if file_sync is True and we processed at least one translation if file_sync and lang and translations_processed: - from django.db import transaction - # Run file sync synchronously after transaction commits to prevent race condition # where commit_pending() commits with default message before file sync task runs # Using transaction.on_commit() ensures: @@ -4601,14 +4597,6 @@ def _do_file_sync_for_autobatchtranslation_immediate( This function reads from database, writes to files, and commits to git. Minimal database modifications are performed (local_revision update). """ - from datetime import UTC - - from django.utils import timezone - - from weblate.trans.exceptions import FileParseError - from weblate.trans.models.pending import PendingUnitChange - from weblate.utils.errors import report_error - # For autobatch translation, handle only one translation: the specified language for this component if not lang: self.log_error( @@ -4616,8 +4604,6 @@ def _do_file_sync_for_autobatchtranslation_immediate( ) return False - from weblate.lang.models import Language - try: language = Language.objects.get(code=lang) except Language.DoesNotExist: @@ -4650,13 +4636,6 @@ def _do_file_sync_for_autobatchtranslation_immediate( # Track translations that were updated and need to be committed translations_to_commit: list[tuple[Translation, str, datetime]] = [] - # Get author name from request if available - author = ( - request.user.get_author_name() - if request and request.user - else "Weblate " - ) - # Write files first (inside lock) with self.repository.lock: # translation.component is already set to self since we fetched it via self.translation_set.get() @@ -4678,8 +4657,6 @@ def _do_file_sync_for_autobatchtranslation_immediate( # Read pending changes for this translation # For autobatch translation, bypass commit policy filtering to force commit # even if units are STATE_FUZZY (which is the default for auto-translations) - from weblate.trans.models.pending import PendingUnitChange - # Simply get all pending changes for this translation # We bypass commit policy filtering to ensure STATE_FUZZY units are committed pending_changes = list( @@ -4692,10 +4669,11 @@ def _do_file_sync_for_autobatchtranslation_immediate( return False # Group changes by author for consistent file writing - commit_groups = translation._group_changes_by_author(pending_changes) - file_updated = False + commit_groups = translation._group_changes_by_author( # noqa: SLF001 + pending_changes + ) - for group_idx, (author_obj, changes) in enumerate(commit_groups, 1): + for author_obj, changes in commit_groups: author_name = author_obj.get_author_name() if author_obj else "Unknown" timestamp = max(change.timestamp for change in changes) @@ -4706,7 +4684,6 @@ def _do_file_sync_for_autobatchtranslation_immediate( changes, store, author_name ) if any(changes_status.values()): - file_updated = True was_changed = True # Track this translation for committing after lock is released @@ -4728,9 +4705,7 @@ def _do_file_sync_for_autobatchtranslation_immediate( # Continue with next group even if one fails # Commit all updated translations (outside lock to avoid deadlock) - for idx, (translation, author_name, commit_timestamp) in enumerate( - translations_to_commit, 1 - ): + for translation, author_name, commit_timestamp in translations_to_commit: component = translation.component # Use default autobatch translation commit message template # Uses template variables that will be rendered by render_template diff --git a/weblate/trans/tasks.py b/weblate/trans/tasks.py index c22457b3c499..4db76c84e2b6 100644 --- a/weblate/trans/tasks.py +++ b/weblate/trans/tasks.py @@ -22,7 +22,7 @@ Count, F, ) -from django.http import Http404, HttpRequest +from django.http import Http404 from django.utils import timezone from django.utils.timezone import make_aware from django.utils.translation import override @@ -819,13 +819,13 @@ def perform_file_sync_for_autobatchtranslation( user_id: int | None = None, ) -> bool: """Write pending changes to files and commit them for autobatch translation.""" - request: HttpRequest | None = None + request: AuthenticatedHttpRequest | None = None if user_id: - request = HttpRequest() + request = AuthenticatedHttpRequest() request.user = User.objects.get(pk=user_id) component = Component.objects.get(pk=pk) - return component._do_file_sync_for_autobatchtranslation_immediate( + return component._do_file_sync_for_autobatchtranslation_immediate( # noqa: SLF001 lang=lang, request=request, ) @@ -845,12 +845,12 @@ def perform_autobatchtranslate_via_openrouter( file_sync: bool = False, ) -> None: """Run autobatch translation via OpenRouter for autobatch translation.""" - request: HttpRequest | None = None + request: AuthenticatedHttpRequest | None = None if user_id: - request = HttpRequest() + request = AuthenticatedHttpRequest() request.user = User.objects.get(pk=user_id) component = Component.objects.get(pk=pk) - component._autobatchtranslate_via_openrouter_immediate( + component._autobatchtranslate_via_openrouter_immediate( # noqa: SLF001 lang=lang, request=request, file_sync=file_sync, diff --git a/weblate/utils/openrouter_translator.py b/weblate/utils/openrouter_translator.py old mode 100755 new mode 100644 index 8240a04a2b00..b07c3100a1c1 --- a/weblate/utils/openrouter_translator.py +++ b/weblate/utils/openrouter_translator.py @@ -1,12 +1,13 @@ -""" -OpenRouter API Translation for Weblate -Batch translation using OpenRouter API via OpenAI SDK. -""" +# Copyright © Boost Organization +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""OpenRouter API translation for Weblate (batch translation via OpenAI SDK).""" from __future__ import annotations import time -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, cast from openai import OpenAI @@ -132,13 +133,20 @@ def translate_batch_json( completion = self.client.chat.completions.create( extra_headers={"X-Title": "Documentation Batch Translation"}, model=self.model, - messages=messages, + messages=cast(Any, messages), # noqa: TC006 response_format={"type": "json_object"}, temperature=0, max_tokens=60000, ) - response_text = completion.choices[0].message.content.strip() + raw_content = completion.choices[0].message.content + if raw_content is None: + self.log_error( + "Batch translation API returned empty message content" + ) + msg = "OpenRouter returned no message content" + raise ValueError(msg) + response_text = raw_content.strip() # Clean up response - remove markdown code fences if present if response_text.startswith("```"): @@ -192,3 +200,6 @@ def translate_batch_json( # For non-429 errors, raise immediately self.log_error("Batch translation API request failed: %s", e) raise + + msg = "OpenRouter batch translation exhausted retries without result" + raise RuntimeError(msg) diff --git a/weblate/utils/quickbook.py b/weblate/utils/quickbook.py index dfbede9a0c58..31d0abeb5a85 100644 --- a/weblate/utils/quickbook.py +++ b/weblate/utils/quickbook.py @@ -215,27 +215,27 @@ def _parse_bracket_keyword(text: str) -> tuple[str, int]: n = len(text) # Single-character special keywords: /, #, $, @, ?, : - if i < n and text[i] in ("/", "#", "$", "@", "?", ":"): + if i < n and text[i] in {"/", "#", "$", "@", "?", ":"}: kw = text[i] i += 1 - while i < n and text[i] in (" ", "\t"): + while i < n and text[i] in {" ", "\t"}: i += 1 return kw, i # Multi-character keyword: read until whitespace, ], or : kw_start = i - while i < n and text[i] not in (" ", "\t", "\n", "]", ":"): + while i < n and text[i] not in {" ", "\t", "\n", "]", ":"}: i += 1 kw = text[kw_start:i].lower() # Optional :id suffix (e.g. ``section:my_anchor``) if i < n and text[i] == ":": i += 1 - while i < n and text[i] not in (" ", "\t", "\n", "]"): + while i < n and text[i] not in {" ", "\t", "\n", "]"}: i += 1 # skip the id token # Skip trailing spaces / tabs after keyword or :id, then one optional newline. - while i < n and text[i] in (" ", "\t"): + while i < n and text[i] in {" ", "\t"}: i += 1 if i < n and text[i] == "\n": i += 1 @@ -456,7 +456,7 @@ def _parse_table_inner( i = inner_abs_start + nl + 1 while i < inner_abs_end: ch = content[i] - if ch in (" ", "\t", "\n"): + if ch in {" ", "\t", "\n"}: i += 1 continue if ch != "[": @@ -472,7 +472,7 @@ def _parse_table_inner( ci = i + 1 # skip the opening '[' while ci < row_end: cc = content[ci] - if cc in (" ", "\t", "\n"): + if cc in {" ", "\t", "\n"}: ci += 1 continue if cc != "[": @@ -556,7 +556,7 @@ def _parse_qbk( # ── code block: line begins with space or tab ───────────────────────── # Only treat as a code block if we are at the very start of a line. - if ch in (" ", "\t") and (i == 0 or content[i - 1] == "\n"): + if ch in {" ", "\t"} and (i == 0 or content[i - 1] == "\n"): # Consume all consecutive indented or blank lines. while i < stop: while i < stop and content[i] != "\n": @@ -566,7 +566,7 @@ def _parse_qbk( i += 1 line += 1 # Stop when the next line is neither blank nor indented. - if i < stop and content[i] not in (" ", "\t", "\n"): + if i < stop and content[i] not in {" ", "\t", "\n"}: break continue @@ -718,7 +718,7 @@ def _parse_qbk( continue # ── tables and variable lists (cell-level parsing) ──────────── - if kw in ("table", "variablelist"): + if kw in {"table", "variablelist"}: segments.extend( _parse_table_inner( content, @@ -747,7 +747,7 @@ def _parse_qbk( if not line_text.strip(): # blank line → end of para break - if line_text and line_text[0] in (" ", "\t"): # code block next + if line_text and line_text[0] in {" ", "\t"}: # code block next break if line_text.startswith("'''"): # raw escape next break @@ -777,7 +777,7 @@ def _parse_qbk( stripped = content[para_start:i].rstrip() if stripped and _has_prose(stripped): first_non_ws = stripped.lstrip()[0] - is_list = first_non_ws in ("*", "#") + is_list = first_non_ws in {"*", "#"} if is_list: # Keep newlines: each line is a structural list item. msgid = stripped @@ -909,6 +909,12 @@ def po_to_qbk(template_content: str, po_store: Any, filename: str) -> str: translation = translations.get(seg.msgid) if translation: + # The original span may end with a newline (e.g. code-fence + # content whose text_end points at the closing ``` line). + # The cleaned msgid/msgstr has no trailing newline, so restore + # it to keep whatever follows (``` or otherwise) on its own line. + if seg.text_end > 0 and template_content[seg.text_end - 1] == "\n": + translation = translation.rstrip("\n") + "\n" parts.append(translation) else: # No translation available: keep the original source text.