Translation - scripts, the final part

.gitlab-ci.yml

@@ -2,5 +2,46 @@ include:
   - project: 'QubesOS/qubes-continuous-integration'
     file: '/gitlab-website.yml'
 
+stages:
+ - prepare
+ - build
+
 build:website:
   extends: .website
+
+update-transifex:
+  tags:
+  - docker
+  stage: prepare
+  rules:
+  - if: '$TX_TOKEN && $GITHUB_KEY'
+    when: always
+  - when: never
+  artifacts:
+    expire_in: 7 days
+    when: always
+    paths:
+    - site.tar.gz
+  variables:
+    GIT_SUBMODULE_STRATEGY: normal
+    GIT_AUTHOR_NAME: translation bot
+    GIT_AUTHOR_EMAIL: builder-bot@qubes-os.org
+    GIT_COMMITTER_NAME: translation bot
+    GIT_COMMITTER_EMAIL: builder-bot@qubes-os.org
+    PAGES_REPO_NWO: QubesOS/qubesos.github.io
+    TRANSLATED_LANGS: de fr es
+    LANG: C.UTF-8
+  before_script:
+  - mkdir -p $HOME/.ssh && echo "$GITHUB_KEY" > $HOME/.ssh/id_ed25519 && chmod 700 $HOME/.ssh/id_ed25519
+  - echo "github.com,140.82.121.4 ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" >> $HOME/.ssh/known_hosts
+  - export PATH=$PATH:$HOME/bin
+  - sudo dnf install -y python3-pycurl python3-PyYAML python3-jsonschema python3-certifi python3-attrs /usr/bin/bundle rubygem-jekyll rubygem-nokogiri rubygem-concurrent-ruby ruby-devel gcc-c++ transifex-client crudini python3-pycurl python3-pyrsistent
+  - pip install python-frontmatter
+  - export NOKOGIRI_USE_SYSTEM_LIBRARIES=true
+  - gem install github-pages json html-proofer
+  - git submodule update --init
+  script:
+  - _utils/transifex-push
+  - _utils/transifex-pull $TRANSLATED_LANGS
+  after_script:
+  - tar czf site.tar.gz _site

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "_hcl"]
 	path = _hcl
 	url = https://github.com/QubesOS/qubes-hcl
+[submodule "_translated"]
+	path = _translated
+	url = https://github.com/QubesOS/qubes-translated

README.md

@@ -126,6 +126,68 @@ Please carefully read these guidelines before submitting a pull request.
  - [jQuery 1.7](http://api.jquery.com) - javascript helper library
  - [jQuery ToC MD Generator](https://github.com/dafi/tocmd-generator) - renders header menu on documentation section
 
+Translation
+------------
+
+Documentation translation is done using Transifex platform: https://www.transifex.com/otf/qubes/
+The `_translated` directory should not be modified manually. Any manual change
+there **will be overriden** with the content downloaded from Transifex.
+
+The `qubes-translated` repository is not signed and generally should not be
+considered trusted for sensitive tasks. But the specific commit referenced from
+this repository is validated to not interfere with English website.
+
+### Transifex integration details ###
+
+Most of the integration is automated. It is split into few parts:
+
+1. `_utils/transifex-push` script takes the source (English) content and
+   uploads to Transifex. The platform merges existing translations to the new
+   files, so unchanged parts that were translated before remain translated.
+   Transifex configuration is created from scratch here, to correctly handle
+   new/removed files.
+
+2. `_utils/transifex-pull` pulls translated content and places into
+   `_translated` submodule. Then a set of scripts in
+   `_utils/_translation_utils` perform various post processing steps,
+   including:
+   - validate syntax of retrieved files (if frontmatter is still correctly set etc)
+   - modify frontmatter settings (`permalink`, `lang`, `redirect_from` etc) to
+     match the page language
+   - adjust all internal links to point at pages in the same language
+   - run htmlproofer to verify if no broken links were introduced
+
+   At the end, the script commit and push the new content to qubes-translated
+   repository.
+
+3. `_utils/update-translated` fetches new version of qubes-translated repo (its
+   master branch), verifies if any page doesn't try to subvert English version,
+   and if all is fine, makes a commit and push updated submodule (similar to
+   `_utils/update-submodules` script).
+
+The points 1 and 2 are running in Gitlab CI environment, without access to any
+signing key and with push access only to qubes-translated repository. The third
+point is running in a more trusted environment, with access to signing key and
+push access to the main repository.
+
+### Language switcher ###
+
+The top level `_config.yml` file contains list of languages to be enabled. If
+there is more than one (`en`), each page will have a language switch menu in
+the top right corner. Only languages listed in `_config.yml` are visible in the
+switcher, but there may be more available (accessing them require manually
+changing language code in the URL).
+
+Each markdown file in the repo has `lang` and `ref` attributes (in its
+frontmatter). `lang` attribute contains the language of this file
+(should always be `en` outside of qubes-translated repository) and `ref`
+contains a unique identifier of that page. Language switcher logic uses the
+`ref` attribute to find all translations of given page. This allows translated
+page to have different page name in URL, although we do not do this right now.
+
+`lang` and `ref` attributes are added with
+`_utils/_translation_utils/prepare_for_translation.py` script.
+
 Deprecated Documentation
 ------------------------
 

_config.yml

@@ -21,6 +21,14 @@ relative_permalinks: false
 permalink: /news/:year/:month/:day/:title/
 excerpt_separator: ""
 
+# enabled languages
+# remember to create symlinks in _data/translation too
+languages:
+- en
+- de
+- fr
+- es
+
 plugins:
   - jekyll-redirect-from
   - jekyll-sitemap
@@ -30,7 +38,7 @@ collections:
     output: true
   hcl:
     output: false
-  qubes-translated:
+  translated:
     output: true
 
 

_data/includes.yml

@@ -11,6 +11,8 @@
       icon: fa-code
     - text: Edit This Page
       icon: fa-code-fork
+    - text: Translate This Page
+      icon: fa-language
     - url: /security/
       text: Report a Security Issue
       icon: fa-lock

_data/translation/de

@@ -0,0 +1 @@
+../../_translated/de/_data/de

_data/translation/es

@@ -0,0 +1 @@
+../../_translated/es/_data/es

_data/translation/fr

@@ -0,0 +1 @@
+../../_translated/fr/_data/fr

_includes/doc-widget.html

@@ -5,9 +5,14 @@
 {% if page.lang == nil or page.lang == "en" %}
   {% assign master_edit =  site.project_repo_path | append: "/qubes-doc/edit/master" | append: page.relative_path | remove_first: '_doc'  %}
   {% assign master_blob =  site.project_repo_path | append: "/qubes-doc/blob/master" | append: page.relative_path | remove_first: '_doc'  %}
+  {% assign pagelang = "" %}
+  {% assign transifexresource = page.relative_path | replace: '_doc/', 'doc/' | remove: ".md" | replace: "/", "_" %}
 {% else %}
-  {% assign master_edit =  site.project_repo_path | append: "/qubes-translated/edit/master" | append: page.relative_path | remove_first: '_qubes-translated'  %}
-  {% assign master_blob =  site.project_repo_path | append: "/qubes-translated/blob/master" | append: page.relative_path| remove_first: '_qubes-translated'  %}
+  {% assign master_edit =  site.project_repo_path | append: "/qubes-translated/edit/master" | append: page.relative_path | remove_first: '_translated'  %}
+  {% assign master_blob =  site.project_repo_path | append: "/qubes-translated/blob/master" | append: page.relative_path | remove_first: '_translated'  %}
+  {% assign pagelang = page.lang %}
+  {% assign prefix = "_translated/" | append: pagelang | append: '/' %}
+  {% assign transifexresource = page.relative_path | remove_first: prefix | replace: '_doc/', 'doc/' | remove: ".md" | replace: "/", "_" %}
 {% endif %}
 <!-- Button links -->
 {% for item in docs.links %}
@@ -19,7 +24,13 @@
   {% if item.icon == "fa-code" %}
     {% assign a_href = master_blob %}
   {% elsif item.icon == "fa-code-fork" %}
+    {% if lang != "" %}
+      <!-- skip "Edit This Page" button on translated pages, link to Transifex only -->
+      {% continue %}
+    {% endif %}
     {% assign a_href = master_edit %}
+  {% elsif item.icon == "fa-language" %}
+    {% assign a_href = "https://www.transifex.com/otf/qubes/translate/#" | append: pagelang | append: "/" | append: transifexresource %}
   {% else %}
     {% assign a_href = item.url %}
   {% endif %}

_includes/header.html

@@ -10,25 +10,37 @@
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </button>
-      <!-- Disable language switcher for now -->
-      {% assign langmenu = false %}
+      <!-- Disable language switcher if there is only one language -->
+      {% if site.languages.size > 1 %}
+        {% assign langmenu = true %}
+      {% else %}
+        {% assign langmenu = false %}
+      {% endif %}
       {% if page.layout == nil or page.collection == 'posts' %}
         <!-- No translation for news -->
         {% assign langmenu = false %}
       {% endif %}
       {% if langmenu %}
-        {% assign posts = site.pages | concat: site.doc | concat: site.qubes-translated | where:'ref', page.ref | sort: 'lang' %}
+        <!-- find this page in other languages -->
+        {% assign page_all_langs = site.pages | concat: site.doc | concat: site.translated | where:'ref', page.ref | sort: 'lang' %}
         <div class="langdropdown langdropdown-sm">
           <label class="langdropbtn-sm" for="langdropdown-toggle-sm"><img src="/attachment/icons/icons8-translation-64.png" width="24" height="24"></label>
           <input type="checkbox" id="langdropdown-toggle-sm">
           <div class="langdropdown-content" id="langdropdown-dd-sm">
-            {% for post in posts %}
-              <a class="c1" href="{{ post.url }}">{{ post.lang }}</a>
+            {% for translated in page_all_langs %}
+              {% if site.languages contains translated.lang %}
+                <a class="c1" href="{{ translated.url }}">{{ translated.lang }}</a>
+              {% endif %}
             {% endfor %}
           </div>
         </div>
       {% endif %}
-      <a class="page-link" href="/">
+      {% if page.lang == nil or page.lang == "" or page.lang == "en" %}
+        {% assign linktohome = "/" %}
+      {% else %}
+        {% assign linktohome = "/" | append: page.lang | append: "/" %}
+      {% endif %}
+      <a class="page-link" href="{{ linktohome }}">
         <img src="/attachment/icons/128x128/apps/qubes-logo-icon.png" width="44" height="44" alt="Qubes OS Project">
         <span><strong>Qubes</strong> OS</span>
       </a>
@@ -46,8 +58,8 @@
               <label class="langdropbtn-lg" for="langdropdown-toggle-lg"><img src="/attachment/icons/icons8-translation-64.png" width="24" height="24"></label>
               <input type="checkbox" id="langdropdown-toggle-lg">
               <div class="langdropdown-content" id="langdropdown-dd-lg">
-                {% for post in posts %}
-                  <a class="c1" href="{{ post.url }}">{{ post.lang }}</a>
+                {% for translated in page_all_langs %}
+                  <a class="c1" href="{{ translated.url }}">{{ translated.lang }}</a>
                 {% endfor %}
               </div>
             </div>

_includes/team.html

@@ -6,7 +6,7 @@
 {% assign emeritus = team-page | where_exp: "item", "item.htmlsection == 'emeritus'" | first %}
 {% assign community = team-page | where_exp: "item", "item.htmlsection == 'community'" | first %}
 {% assign team_link = lang | append: "/team/" %}
-{% assign teams = site.pages | concat: site.qubes-translated | where:'permalink', team_link %}
+{% assign teams = site.pages | concat: site.translated | where:'permalink', team_link %}
 {% if teams.size == 0 %}
   {% assign team_link = "/team/" %}
 {% endif %}

_utils/_translation_utils/COUNTER.txt

@@ -0,0 +1 @@
+current counter: 251

_utils/_translation_utils/check_all_langs.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# to be run from the git root
+# $1 is directory where translated files reside and language needs to be added to internal urls
+# TODO param check
+
+set -e
+
+echo "================================= build site =================================="
+#read b
+bundle exec jekyll b
+
+all_ok=true
+echo "================================= run htmlproofer ==============================="
+htmlproofer ./_site   --disable-external   --checks-to-ignore ImageCheck   --file-ignore "./_site/video-tours/index.html,./_site/.*/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug 2&> /tmp/html.output || all_ok=false
+
+# exit here if all is ok
+if $all_ok; then
+    echo 'All checks passed!'
+    exit
+fi
+
+echo "================================== as a last resort in case of errors process html proofer errors ================================="
+python3 _utils/_translation_utils/postprocess_htmlproofer.py /tmp/html.output "$1"
+
+echo "================================= build the site and run htmlproofer ===================================="
+rm -rf ./_site/
+bundle exec jekyll b
+htmlproofer ./_site   --disable-external   --checks-to-ignore ImageCheck   --file-ignore "./_site/video-tours/index.html,./_site/.*/video-tours/index.html" --url-ignore "/qubes-issues/" --log-level debug