diff --git a/.github/scripts/checkTranslation.py b/.github/scripts/checkTranslation.py index 7ca92cd..b067d6d 100644 --- a/.github/scripts/checkTranslation.py +++ b/.github/scripts/checkTranslation.py @@ -2,93 +2,124 @@ import os from crowdin_api import CrowdinClient -# ----------------------------- -# CROWDIN API SCORE -# ----------------------------- +def find_file_id(client, project_id, base_target, search_ext): + """ + Iterates through all project files (using pagination) to find the ID + of the source file matching the target name and extension. + """ + offset = 0 + limit = 100 + + while True: + resp = client.source_files.list_files( + projectId=project_id, + limit=limit, + offset=offset + ) + + data = resp['data'] + for f in data: + path_crowdin = f['data']['path'].lower() + # Check if the path ends with addon_id.pot or addon_id.xliff + if path_crowdin.endswith(f"{base_target}{search_ext}"): + file_id = f['data']['id'] + print(f"DEBUG: Match found: {path_crowdin} (ID: {file_id})") + return file_id + + if len(data) < limit: + break + + offset += limit + return None -def get_score_from_api(lang_id: str, crowdin_file_name: str) -> float: +def get_score_from_api(file_name_to_search: str, lang_id: str) -> float: """ - Fetches the translation progress percentage directly from Crowdin API. - Returns a float between 0.0 and 1.0. + Retrieves the translation progress score for a specific language and file. + Handles pagination for both file listing and language status. """ token = os.environ.get("crowdinAuthToken") - project_id_env = os.environ.get("CROWDIN_PROJECT_ID") + p_id_env = os.environ.get("CROWDIN_PROJECT_ID") - # Ensure credentials are present - if not token or not project_id_env: + if not token or not p_id_env: + print("ERROR: Missing environment variables 'crowdinAuthToken' or 'CROWDIN_PROJECT_ID'.") return 0.0 client = CrowdinClient(token=token) - project_id = int(project_id_env) + p_id = int(p_id_env) try: - # 1. NORMALIZE SEARCH TERMS - # Extract base name (e.g., 'askOpenRouter') and extension - base_target = crowdin_file_name.replace("\\", "/").split("/")[-1].rsplit(".", 1)[0].lower() - ext_target = crowdin_file_name.split(".")[-1].lower() + # Clean and prepare search patterns + # Example: 'addon/locale/fr/LC_MESSAGES/myaddon.po' -> base_target: 'myaddon' + base_target = file_name_to_search.replace('\\', '/').split('/')[-1].rsplit('.', 1)[0].lower() + ext_target = file_name_to_search.split('.')[-1].lower() - # Mapping: if we check a .po, we look for a .pot on Crowdin + # On Crowdin, the source for a .po file is usually a .pot file search_ext = ".pot" if ext_target == "po" else f".{ext_target}" - # 2. FETCH ALL FILES TO FIND MATCHING ID - files = client.source_files.with_fetch_all().list_files(project_id) - file_id = None + print(f"DEBUG: Searching for source file: {base_target}{search_ext}") - for f in files["data"]: - path_crowdin = f["data"]["path"].lower() - if path_crowdin.endswith(f"{base_target}{search_ext}"): - file_id = f["data"]["id"] - break + file_id = find_file_id(client, p_id, base_target, search_ext) if file_id is None: + print(f"WARNING: File '{base_target}{search_ext}' not found on Crowdin.") return 0.0 - # 3. FETCH PROGRESS FOR THE SPECIFIC FILE - # We use get_file_progress which is reliable for specific file IDs - progress = client.translation_status.get_file_progress(projectId=project_id, fileId=file_id) - - for item in progress["data"]: - if item["data"]["languageId"].lower() == lang_id.lower(): - # Return ratio (0.0 to 1.0) - return float(item["data"]["translationProgress"]) / 100 + # Pagination for translation status (Progress) + offset = 0 + limit = 100 + + while True: + resp = client.translation_status.get_file_progress( + projectId=p_id, + fileId=file_id, + limit=limit, + offset=offset + ) + + data = resp['data'] + for item in data: + lang_api = item['data']['languageId'] + + # Flexible matching (e.g., 'fr' will match 'fr' or 'fr-FR' from API) + # Also handles underscore to dash conversion for Crowdin compatibility + if lang_api.lower().startswith(lang_id.lower().replace('_', '-')): + progress = float(item['data']['translationProgress']) + return progress / 100 + + # Check pagination total + total = resp['pagination']['totalCount'] + if offset + limit >= total: + break + offset += limit - except Exception: - # Fallback to 0.0 in case of API or network error + print(f"DEBUG: Language '{lang_id}' not found in progress list for this file.") return 0.0 - return 0.0 - - -# ----------------------------- -# MAIN ENGINE -# ----------------------------- - + except Exception as e: + print(f"API ERROR: {e}") + return 0.0 def main(): - """ - Main entry point. - Expects two arguments: - """ if len(sys.argv) < 3: + print("Usage: python checkTranslation.py ") sys.exit(2) - file_name = sys.argv[1] + input_file = sys.argv[1] lang = sys.argv[2] - # All evaluations now go through the Crowdin API - score = get_score_from_api(lang, file_name) + score = get_score_from_api(input_file, lang) - # Output formatting for PowerShell capture + # Output formatted for capture by the PowerShell script (crowdinSync.ps1) print(f"translationRatio={score}") - if file_name.lower().endswith(".md"): + + if input_file.lower().endswith('.md'): print(f"mdScore={score}") else: print(f"poScore={score}") - # Exit with code 0 if score > 5%, otherwise 1 + # Exit with success (0) if there is at least some translated content sys.exit(0 if score > 0.05 else 1) - if __name__ == "__main__": main() diff --git a/.github/scripts/crowdinSync.ps1 b/.github/scripts/crowdinSync.ps1 index 0237ef1..a96ae3e 100644 --- a/.github/scripts/crowdinSync.ps1 +++ b/.github/scripts/crowdinSync.ps1 @@ -1,7 +1,7 @@ #!/usr/bin/env pwsh $ErrorActionPreference = 'Stop' -# Config git +# Git configuration for automated commits git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" @@ -11,133 +11,156 @@ if (-not $addonId) { exit 1 } -# Update xliff file +# --- STEP 1: PREPARATION AND SOURCE UPDATE --- + $xliffFile = "./$addonId.xliff" $mdFile = "./readme.md" + if (Test-Path $mdFile) { if (Test-Path $xliffFile) { $tempXliff = [System.IO.Path]::GetTempFileName() Copy-Item "$addonId.xliff" $tempXliff -Force - Write-Host "Copied $addonId.xliff to temporary file: $tempXliff" + Write-Host "DEBUG: Updating XLIFF source based on readme.md..." uv run .github/scripts/markdownTranslate.py updateXliff -m $mdFile -x $tempXliff -o $xliffFile - Write-Host "Updated $xliffFile based on $mdFile" } else { - Write-Host "XLIFF file not found, but readme.md exists. Creating an XLIFF template for translations." + Write-Host "DEBUG: XLIFF template not found. Creating new one from readme.md..." uv run .github/scripts/markdownTranslate.py generateXliff -m $mdFile -o $xliffFile } -} else { - Write-Host "readme.md not found. Skipping XLIFF generation." } -# Update pot file in Crowdin +# Update POT file (addon interface) uv run scons pot $potFile = "$addonId.pot" + +# --- STEP 2: UPLOAD SOURCES TO CROWDIN --- + if (Test-Path $potFile) { - Write-Host "Uploading updated POT to Crowdin..." + Write-Host "DEBUG: Uploading updated POT source to Crowdin..." ./l10nUtil.exe uploadSourceFile "$potFile" -c addon -} else { - Write-Host "POT file not found, skipping POT update." } -# Update xliff file in Crowdin if (Test-Path $xliffFile) { - Write-Host "Uploading XLIFF to Crowdin..." + Write-Host "DEBUG: Uploading updated XLIFF source to Crowdin..." ./l10nUtil.exe uploadSourceFile "$xliffFile" -c addon git add "$xliffFile" git diff --staged --quiet if ($LASTEXITCODE -ne 0) { git commit -m "Update $xliffFile for $addonId" git push - } else { - Write-Host "No changes to $xliffFile, skipping commit." } -} else { - Write-Host "XLIFF file not found, skipping XLIFF upload." } -# Export translations -Write-Host "Exporting translations from Crowdin..." +# --- STEP 3: EXPORT AND PROCESS TRANSLATIONS --- + +Write-Host "DEBUG: Exporting translations from Crowdin..." ./l10nUtil.exe exportTranslations -o _addonL10n -c addon # Ensure base directories exist New-Item -ItemType Directory -Force -Path addon/locale | Out-Null New-Item -ItemType Directory -Force -Path addon/doc | Out-Null +# Load language mappings for Crowdin API calls $languageMappings = Get-Content -Raw ".github/scripts/languageMappings.json" | ConvertFrom-Json -foreach ($dir in Get-ChildItem -Path "_addonL10n/$addonId" -Directory) { - $langCode = $dir.Name +foreach ($dir in Get-ChildItem -Path "_addonL10n/$addonId" -Directory) { + $langCode = $dir.Name + if ($langCode -eq "en") { continue } + + # Identify codes $crowdinLang = $languageMappings[$langCode] if (-not $crowdinLang) { $crowdinLang = $langCode } - $langShort = $langCode.Split('_')[0] - Write-Host "--- Processing: $addonId ($langCode) ---" + $langShort = $langCode.Split('-')[0].Split('_')[0] - # Temporary files from Crowdin + # Map to local NVDA directory + $localLangDir = uv run python .github/scripts/langCodes.py $langCode + + Write-Host "`n--- Processing Language: $langCode (Mapped to local: $localLangDir) ---" + + # Paths $remoteMd = Join-Path $dir.FullName "$addonId.md" $remoteXliff = Join-Path $dir.FullName "$addonId.xliff" $remotePo = Join-Path $dir.FullName "$addonId.po" - - # Local paths - $localMdDir = "addon/doc/$langCode" + $localMdDir = "addon/doc/$localLangDir" $localMd = "$localMdDir/readme.md" - $localPoPath = "addon/locale/$langCode/LC_MESSAGES/nvda.po" + $localPoPath = "addon/locale/$localLangDir/LC_MESSAGES/nvda.po" - # 1. PO PROCESSING + # --- 3.1 PO FILE PROCESSING --- + $poImported = $false if (Test-Path $remotePo) { + Write-Host "DEBUG: Checking Remote PO progress for $langShort..." uv run python .github/scripts/checkTranslation.py "$addonId.po" $langShort if ($LASTEXITCODE -eq 0) { + Write-Host "SUCCESS: Remote PO is valid. Importing to $localPoPath" New-Item -ItemType Directory -Force -Path (Split-Path $localPoPath) | Out-Null Move-Item $remotePo $localPoPath -Force + $poImported = $true + } else { + Write-Host "WARNING: Remote PO progress is below threshold." } } - # 2. EVALUATION VIA API + if (-not $poImported -and (Test-Path $localPoPath)) { + Write-Host "ACTION: Uploading local legacy PO to Crowdin ($crowdinLang) as fallback." + ./l10nUtil.exe uploadTranslationFile $crowdinLang "$addonId.po" $localPoPath -c addon + } + + # --- 3.2 DOCUMENTATION PROCESSING (MD & XLIFF) --- $scoreMd = 0.0 $scoreXliff = 0.0 if (Test-Path $remoteMd) { + Write-Host "DEBUG: Evaluating Remote Markdown score..." $res = uv run python .github/scripts/checkTranslation.py "$addonId.md" $langShort $scoreMd = [double]($res | Select-String "mdScore=").ToString().Split("=")[1] + } else { + Write-Host "DEBUG: No remote Markdown file found for this language." } if (Test-Path $remoteXliff) { + Write-Host "DEBUG: Evaluating Remote XLIFF score..." $res = uv run python .github/scripts/checkTranslation.py "$addonId.xliff" $langShort $scoreXliff = [double]($res | Select-String "translationRatio=").ToString().Split("=")[1] + } else { + Write-Host "DEBUG: No remote XLIFF file found for this language." } - Write-Host "Scores -> MD: $scoreMd | XLIFF: $scoreXliff" + Write-Host "DEBUG: Comparison Scores -> MD: $scoreMd | XLIFF: $scoreXliff" - # 3. DECISION LOGIC $threshold = 0.5 - $imported = $false + $docImported = $false if ($scoreXliff -gt $threshold -or $scoreMd -gt $threshold) { - # Create doc directory if needed if (!(Test-Path $localMdDir)) { New-Item -ItemType Directory -Force -Path $localMdDir | Out-Null } if ($scoreXliff -ge $scoreMd) { - Write-Host "Action: Converting XLIFF to local MD" + Write-Host "SUCCESS: XLIFF is better or equal. Converting XLIFF to local MD ($localLangDir)..." ./l10nUtil.exe xliff2md $remoteXliff $localMd - $imported = $true + $docImported = $true } else { - Write-Host "Action: Importing Remote MD to local" + Write-Host "SUCCESS: Markdown is better. Importing Remote MD to local ($localLangDir)..." Move-Item $remoteMd $localMd -Force - $imported = $true + $docImported = $true } + } else { + Write-Host "WARNING: Both remote MD and XLIFF scores are below threshold ($threshold)." } - # 4. FALLBACK: Upload local if remote is poor - if (-not $imported -and (Test-Path $localMd)) { - Write-Host "Action: Remote quality too low. Uploading local MD to Crowdin..." + if (-not $docImported -and (Test-Path $localMd)) { + Write-Host "ACTION: Documentation quality too low. Uploading local MD to Crowdin ($crowdinLang) as fallback." ./l10nUtil.exe uploadTranslationFile $crowdinLang "$addonId.md" $localMd -c addon } } +# --- STEP 4: COMMIT UPDATED TRANSLATIONS --- + git add addon/locale addon/doc git diff --staged --quiet if ($LASTEXITCODE -ne 0) { - git commit -m "Update translations for $addonId from Crowdin" + git commit -m "Update translations for $addonId from Crowdin (Automatic Sync)" $branch = $env:downloadTranslationsBranch git push -f origin "HEAD:$branch" + Write-Host "SUCCESS: Translations committed and pushed." +} else { + Write-Host "DEBUG: No changes in translations to commit." } diff --git a/.github/scripts/langCodes.py b/.github/scripts/langCodes.py new file mode 100644 index 0000000..044e5b5 --- /dev/null +++ b/.github/scripts/langCodes.py @@ -0,0 +1,60 @@ +import sys + +# Mapping between Crowdin language IDs (keys) and standard NVDA directory names (values). +# This dictionary acts as the symmetrical counterpart to 'languageMappings.json' implemented by @nvdaes. +# It ensures that translations exported from Crowdin are stored in the correct +# local paths (e.g., 'es-ES' from Crowdin goes into the 'es' folder). +CROWDIN_TO_NVDA = { + # Arabic variants + "ar-SA": "ar_SA", + + # Spanish variants + "es-ES": "es", + "es-CO": "es_CO", + + # Portuguese variants + "pt-BR": "pt_BR", + "pt-PT": "pt_PT", + + # Chinese variants + "zh-CN": "zh_CN", + "zh-HK": "zh_HK", + "zh-TW": "zh_TW", + + # Other specific mappings from the NVDA ecosystem + "af": "af_ZA", + "de-CH": "de_CH", + "nb": "nb_NO", + "nn-NO": "nn_NO", + "sr-CS": "sr" +} + +def get_nvda_code(crowdin_code): + """ + Returns the appropriate local directory name for a given Crowdin language ID. + + Args: + crowdin_code (str): The language identifier from Crowdin (e.g., 'pt-BR', 'fr'). + + Returns: + str: The corresponding NVDA locale folder name (e.g., 'pt_BR', 'fr'). + """ + # 1. Direct check in our verified map (Priority) + if crowdin_code in CROWDIN_TO_NVDA: + return CROWDIN_TO_NVDA[crowdin_code] + + # 2. Automated conversion for regional variants: Crowdin "xx-YY" -> NVDA "xx_YY" + # This handles regional codes not explicitly defined in the map. + if "-" in crowdin_code: + return crowdin_code.replace("-", "_") + + # 3. Default: Return as is. + # This covers base languages that don't use regional folders in NVDA + # (e.g., 'fr', 'tr', 'bg', 'fi', 'fa'). + return crowdin_code + +if __name__ == "__main__": + # Ensure a language code was provided as a command-line argument + if len(sys.argv) > 1: + # Standardize input and output the mapped code for PowerShell to capture + print(get_nvda_code(sys.argv[1])) \ No newline at end of file diff --git a/readme.md b/readme.md index c8d33d2..b767408 100644 --- a/readme.md +++ b/readme.md @@ -177,13 +177,28 @@ If not, leave the dictionary empty. ### Translation workflow -You can add the documentation and interface messages of your add-on to be translated in Crowdin. - -You need a Crowdin account and an API token with permissions to push to a Crowdin project. -For example, you may want to use this [Crowdin project to translate NVDA add-ons](https://crowdin.com/project/nvdaaddons). - -Then, to export your add-on to Crowdin for the first time, run the `.github/workflows/exportAddonsToCrowdin.yml`, ensuring that the update option is set to false. -When you have updated messages or documentation, run the workflow setting update to true (which is the default option). +This template allows you to automate the synchronization of documentation and interface messages with Crowdin. + +#### 1. Crowdin Project Setup +You need a Crowdin account and an API token with permissions to manage a project. +If you wish to use the community project [Crowdin project to translate NVDA add-ons](https://crowdin.com/project/nvdaaddons): +* **Request Access:** Send a message to the [NVDA translation mailing list](https://groups.io/g/nvda-translations) (**nvda-translations@groups.io**) requesting an invitation to join the project as a developer. +* **API Token:** Once invited, generate an API token in your Crowdin account settings. + +#### 2. GitHub Secrets +To allow the workflows to communicate with Crowdin, you must add the following secrets to your GitHub repository (`Settings > Secrets and variables > Actions`): +* `crowdinAuthToken`: Paste your Crowdin API token here. +* `CROWDIN_PROJECT_ID`: The ID of your project on Crowdin. + +#### 3. Infrastructure +Ensure that your repository includes the following files (provided in this template): +* **Workflows:** `.github/workflows/exportAddonsToCrowdin.yml` and `.github/workflows/downloadTranslations.yml`. +* **Scripts:** The `.github/scripts/` folder containing `checkTranslation.py`, `langCodes.py`, `languageMappings.json`, and `crowdinSync.ps1`. + +#### 4. Running the Workflow +* **Initial Export:** To export your add-on to Crowdin for the first time, run the `exportAddonsToCrowdin.yml` workflow, ensuring that the "update" option is set to **false**. +* **Updates:** When you have updated messages or documentation, run the same workflow with "update" set to **true** (default). +* **Download:** The `downloadTranslations.yml` workflow will periodically (or manually) fetch new translations, verify their quality using the scripts, and create a Pull Request with the updated `.po` and `readme.md` files. ### Additional tools