Skip to content
Browse files

ENHANCEMENT getlocalization build support

Squashed commits (sorry, too hard to untangle):
- Moved descriptions from custom "phing help" target
to the more standard 'phing -l' command, which keeps
the descriptions in one place, rather than duplicating
them between xml comments, "description" attrs and the "help" target.
- Prefixed helper targets to make it clear that they're internal
(= require temporary properties to work)
  • Loading branch information...
1 parent d5b72a8 commit 00f66e201f5b7ee1a4cd4118f2236008ddfeeea3 @chillu chillu committed Jun 22, 2012
Showing with 347 additions and 41 deletions.
  1. +6 −0 build.properties.default
  2. +114 −41 build.xml
  3. +227 −0 tools/UpdateTranslationsTask.php
View
6 build.properties.default
@@ -0,0 +1,6 @@
+getlocalization.framework.project = sapphire
+getlocalization.framework.user = silverstripe
+getlocalization.framework.password =
+getlocalization.cms.project = silverstripe_cms
+getlocalization.cms.user = silverstripe
+getlocalization.cms.password =
View
155 build.xml
@@ -16,6 +16,7 @@ phing help
<taskdef name="ssmodules" classname="tools.LoadModulesTask" classpath="${project.basedir}" />
<taskdef name="sschanglog" classname="tools.CreateChangelog" classpath="${project.basedir}" />
<taskdef name="gitstash" classname="tools.GitStashTask" classpath="${project.basedir}" />
+ <taskdef name="updateTranslationsTask" classname="tools.UpdateTranslationsTask" classpath="${project.basedir}" />
<property name="basedir" value="." override="true" />
<property name="dependent-modules-file" value="dependent-modules" override="true" />
@@ -28,22 +29,19 @@ phing help
<available file="${dependent-modules-file}" property="dependent-modules-file-exists" />
<available file="${changelog-definition-file}" property="changelog-definition-file-exists" />
+ <!--
+ =================================================================
+ Helper Targets
+ =================================================================
+ -->
+
<target name="help">
<echo>
SilverStripe Project Build
------------------------------------
This build file contains targets to assist in creating new SilverStripe builds and releases.
-
-Important targets:
-
-* archive - Creates a tar.gz or zip file from the current source, removing version control specific files
-* checkout - Switches all working copies to the specified tag or branch
-* tag - Creates a new git tag in all the nested working copies (optionally pushes the created tag)
-* pushtags - Pushes all local tags to their respective origin repositories
-* update_modules - Checks out repositories defined in the 'dependent-modules' file into the current directory
-* add_module - Checks out a module at a specific repository URL
-* changelog - Create a changelog.md file from the repositories specified in the 'changelog-definitions' file
+Run "phing -l" to get a full list of available targets.
Options:
@@ -65,17 +63,16 @@ Options:
</echo>
</target>
- <target name="gitRepositories">
+ <target name="_gitRepositories">
<findRepos TargetDir="${basedir}" />
</target>
- <!-- find the git binary and set it -->
- <target name="gitBinary">
+ <target name="_gitBinary" description="find the git binary and set it">
<exec command="which git" outputProperty="gitPath1" />
</target>
- <!-- Tag a git repo with a specific tag (in $tagname) -->
- <target name="tagTask" if="tagname,reponame,gitPath1">
+ <target name="_tagTask" if="tagname,reponame,gitPath1"
+ description="Tag a git repo with a specific tag.">
<gittag
repository="${reponame}"
name="${tagname}"
@@ -84,21 +81,21 @@ Options:
<echo msg="git tag '${tagname}' added to '${reponame}' git repository" />
</target>
- <!-- Push all local tags -->
- <target name="pushTask" if="reponame,gitPath1">
- <gitpush
+ <target name="_pushTask" if="reponame,gitPath1"
+ description="Push all local tags">
+ <ssgitpush
repository="${reponame}"
tags="true"
gitPath="${gitPath1}" />
<echo msg="pushed all tags to '${reponame}' git repository" />
</target>
- <!-- Checkout the specified tag on all working copies -->
- <target name="checkoutTask" if="reponame,gitPath1,tagname">
+ <target name="_checkoutTask" if="reponame,gitPath1,tagname"
+ description="Checkout the specified tag on all working copies">
<echo msg="checking out ${reponame}"/>
<gitstash repository="${reponame}" gitPath="${gitPath1}" />
- <gitcheckout
+ <ssgitcheckout
repository="${reponame}"
branchname="${tagname}"
gitPath="${gitPath1}" />
@@ -108,17 +105,23 @@ Options:
<echo msg="checked out ${tagname} tag/branch in '${reponame}' git repository" />
</target>
- <target name="createDependentModulesFile" unless="dependent-modules-file-exists">
+ <target name="_createDependentModulesFile" unless="dependent-modules-file-exists">
<copy file="${dependent-modules-file}.default" tofile="${dependent-modules-file}" />
</target>
- <target name="createChangelogDefinitionsFile" unless="changelog-definitions-file-exists">
+ <target name="_createChangelogDefinitionsFile" unless="changelog-definitions-file-exists">
<copy file="${changelog-definitions-file}.default" tofile="${changelog-definitions-file}" />
</target>
+ <!--
+ =================================================================
+ Main Targets
+ =================================================================
+ -->
- <!-- tags all git repositories in the current directory with a tag name -->
- <target name="tag" if="basedir" depends="gitRepositories,gitBinary">
+ <target name="tag" if="basedir"
+ description="Creates a new git tag in all the nested working copies (optionally pushes the created tag)"
+ depends="_gitRepositories,_gitBinary">
<if>
<isset property="tagname"/>
<then>
@@ -131,7 +134,7 @@ Options:
</if>
<!-- find all git repos and run the tagTask on them -->
- <foreach list="${GitReposList}" param="reponame" target="tagTask" />
+ <foreach list="${GitReposList}" param="reponame" target="_tagTask" />
<input propertyName="pushToOrigin" defaultValue="no" validArgs="yes,no" promptChar=":">Push local tags to origin?</input>
<if>
@@ -142,13 +145,15 @@ Options:
</if>
</target>
- <!-- Pushes all local tags to origin -->
- <target name="pushtags" if="basedir" depends="gitRepositories,gitBinary">
- <foreach list="${GitReposList}" param="reponame" target="pushTask" />
+ <target name="pushtags" if="basedir"
+ description="Pushes all local tags to their respective origin repositories"
+ depends="_gitRepositories,_gitBinary">
+ <foreach list="${GitReposList}" param="reponame" target="_pushTask" />
</target>
- <!-- Switches all working copies to the specified tag or branch -->
- <target name="checkout" if="basedir" depends="gitRepositories,gitBinary">
+ <target name="checkout" if="basedir"
+ description="Switches all working copies to the specified tag or branch"
+ depends="_gitRepositories,_gitBinary">
<if>
<isset property="tagname"/>
<then>
@@ -179,7 +184,7 @@ Options:
</if>
<!-- find all git repos and run the checkoutTask on them -->
- <foreach list="${GitReposList}" param="reponame" target="checkoutTask" />
+ <foreach list="${GitReposList}" param="reponame" target="_checkoutTask" />
</target>
<target name="archive" if="basedir" description="Creates a gzip archive from the current folder (removes any version control files)">
@@ -325,7 +330,8 @@ Options:
<echo msg="##teamcity[publishArtifacts '${archivename}-framework-v${version}.zip']" />
</target>
- <target name="upload-release">
+ <target name="upload-release"
+ description="Uploads archives previously created through 'ping archive' to a public webhost, and notifies a group of people of the new release. Requires working public key auth on the release destination.">
<if>
<not><isset property="version"/></not>
<then><input propertyName="version" defaultValue="x.y.z" promptChar=":">Please choose a version</input></then>
@@ -350,22 +356,22 @@ Your friendly automated release script.
</target>
- <target name="upload-nightly">
+ <target name="upload-nightly"
+ description="Uploads archives previously created through 'phing archive' to a public webhost">
<property name="nightly_dest" value="qa-servers@homer:/sites/ssorg-v2/www/assets/nightlies/" />
<exec command="scp -P 2222 SilverStripe-*.tar.gz ${nightly_dest}" />
<exec command="scp -P 2222 SilverStripe-*.zip ${nightly_dest}" />
</target>
- <!-- Load modules where sensitive dependency exists -->
- <target name="update_modules" depends="createDependentModulesFile">
+ <target name="update_modules"
+ description="Checks out repositories defined in the 'dependent-modules' file into the current directory"
+ depends="_createDependentModulesFile">
<ssmodules file="${basedir}/${dependent-modules-file}" noninteractive="${ni_build}"/>
</target>
- <!-- Add a new module to the system. Run from the commandline, you can pass
- in the details of the module as phing add_module -Dmodule=blog -Dmodurl=http://path/to/svn -->
- <target name="add_module">
+ <target name="add_module" description="Checks out a module at a specific repository URL. Usage: phing add_module -Dmodule=blog -Dmodurl=http://path/to/svn.">
<if>
<isset property="modurl"/>
<then>
@@ -391,10 +397,77 @@ Your friendly automated release script.
<ssmodules name="${module}" url="${modurl}" />
</target>
- <!-- Create a changelog of git changes based on the -->
- <target name="changelog" depends="createChangelogDefinitionsFile" if="basedir,changelogSort">
+ <target name="changelog"
+ description="Create a changelog.md file from the repositories specified in the 'changelog-definitions' file"
+ depends="_createChangelogDefinitionsFile"
+ if="basedir,changelogSort">
<sschanglog definitions="${changelog-definitions-file}" baseDir="${basedir}" sort="${changelogSort}"/>
<echo msg="${changelogOutput}" />
</target>
+ <target name="translations-staging-setup">
+ <exec command="git checkout master" dir="${module}" checkreturn="true" />
+ <exec command="git fetch origin" dir="${module}" checkreturn="true" />
+ <exec command="git branch -D translation-staging" dir="${module}" checkreturn="true" />
+ <exec command="git branch --track translation-staging origin/translation-staging" dir="${module}" checkreturn="true" />
+ <exec command="git checkout translation-staging" dir="${module}" checkreturn="true" />
+ </target>
+
+ <target name="translations-staging-teardown">
+ <exec command="git checkout master" dir="${module}" checkreturn="true" />
+ </target>
+
+ <target name="translations-update-git-masterfile"
+ description="Collect translation on a module, commit them into a specialized branch and push to the origin repository."
+ depends="translations-staging-setup">
+ <exec command="git merge origin/master" dir="${module}" checkreturn="true" />
+ <exec command="php framework/cli-script.php dev/tasks/i18nTextCollectorTask" passthru="true" />
+ <exec command="git status --short" dir="${module}" outputProperty="git.status.${module}" />
+ <if>
+ <istrue value="${git.status.${module}}" />
+ <then>
+ <exec command="git add lang/*" dir="${module}" passthru="true" checkreturn="true" />
+ <exec command="git commit -m 'MINOR Updated translations master'" dir="${module}" passthru="true" checkreturn="true" />
+ <exec command="git merge --force origin/master" dir="${module}" checkreturn="true" />
+ </then>
+ </if>
+ </target>
+
+ <target name="translations-update-gl-masterfile"
+ description="Pushes translation master files to getlocalization.com"
+ depends="translations-staging-setup">
+ <exec command="curl --fail --form file=@${module}/lang/en.yml --form name='lang/en.yml' --user ${getlocalization.${module}.user}:${getlocalization.${module}.password} https://api.getlocalization.com/${getlocalization.${module}.project}/api/update-master/" passthru="true" checkreturn="true" />
+ <exec command="git checkout master" dir="${module}" checkreturn="true" />
+ </target>
+
+ <target name="translations-update-gl-contribs" description="Update translations in working copy from getlocalization.com, and commit changes to a specialized branch and push to origin repository. Note: The API requests can take a couple of minutes."
+ depends="translations-staging-setup">
+ <exec command="git stash" dir="${module}" checkreturn="true" />
+ <updateTranslationsTask
+ glProductName="${getlocalization.${module}.project}"
+ glUser="${getlocalization.${module}.user}"
+ glPassword="${getlocalization.${module}.password}"
+ modulePath="${module}"
+ />
+ <exec command="git status --short" dir="${module}" outputProperty="git.status.${module}" />
+ <if>
+ <istrue value="${git.status.${module}}" />
+ <then>
+ <exec command="git add lang/*" dir="${module}" passthru="true" checkreturn="true" />
+ <exec command="git add javascript/lang/*" dir="${module}" passthru="true" checkreturn="true" />
+ <exec command="git commit -m 'MINOR Updated translations'" dir="${module}" passthru="true" checkreturn="true" />
+ <exec command="git push origin/translation-staging" dir="${module}" checkreturn="true" />
+ </then>
+ </if>
+ <exec command="git checkout master" dir="${module}" checkreturn="true" />
+ <exec command="git stash pop" dir="${module}" checkreturn="true" />
+ </target>
+
+ <target name="translations-sync"
+ description="Wrapper task to handle updating translations and master files, using the getlocalization.com API, committing to a specialized branch in the working copy and pushing to the origin repository.">
+ <foreach list="cms" param="module" target="translations-update-git-masterfile" />
+ <foreach list="cms" param="module" target="translations-update-gl-masterfile" />
+ <foreach list="cms" param="module" target="translations-update-gl-contribs" />
+ </target>
+
</project>
View
227 tools/UpdateTranslationsTask.php
@@ -0,0 +1,227 @@
+<?php
+/**
+ * Fetch translation files as a ZIP from getlocalization.com,
+ * extract the YAML files from it, and do some reformatting on them
+ * to make them suitable for usage in the SilverStripe translation system.
+ *
+ * Requires 'unzip' binary.
+ */
+class UpdateTranslationsTask extends SilverStripeBuildTask {
+
+ static $url_translations = 'https://api.getlocalization.com/%s/api/translations/zip/';
+
+ /**
+ * Absolute path to module base (not lang folder)
+ */
+ protected $modulePath;
+
+ /**
+ * GetLocalization product name, as documented in their API:
+ * http://www.getlocalization.com/library/api/get-localization-file-management-api/
+ * @var [type]
+ */
+ protected $glProductName;
+
+ protected $glUser;
+
+ protected $glPassword;
+
+ protected $langFolders = array(
+ 'yml' => 'lang',
+ 'js' => 'javascript/lang'
+ );
+
+ /**
+ * @var String If set, will use existing files rather than try to download them.
+ */
+ protected $downloadPath;
+
+ public function getGlUser() {
+ return $this->glUser;
+ }
+
+ public function setGlUser($newGlUser) {
+ $this->glUser = $newGlUser;
+ return $this;
+ }
+
+ public function getGlPassword() {
+ return $this->glPassword;
+ }
+
+ public function setGlPassword($newGlPassword) {
+ $this->glPassword = $newGlPassword;
+ return $this;
+ }
+
+ public function setModulePath($path) {
+ $this->modulePath = $path;
+ }
+
+ public function setDownloadPath($path) {
+ $this->downloadPath = $path;
+ }
+
+ public function setGlProductName($name) {
+ $this->glProductName = $name;
+ }
+
+ public function main() {
+ if (!is_dir($this->modulePath)) {
+ throw new BuildException("Invalid target directory: $this->modulePath");
+ }
+
+ $downloadPath = $this->downloadPath ? $this->downloadPath : $this->download();
+ $files = $this->findFiles($downloadPath);
+ foreach($files as $file) {
+ $ext = pathinfo($file, PATHINFO_EXTENSION);
+ if($ext == 'yml') {
+ $this->processYmlFile($file);
+ } elseif($ext == 'js') {
+ $this->processJavascriptFile($file);
+ } else {
+ throw new LogicException(sprintf('Unknown extension: %s', $ext));
+ }
+ }
+
+ }
+
+ /**
+ * @return File path to a folder structure containing translation files
+ */
+ protected function download() {
+ $tmpFolder = tempnam(sys_get_temp_dir(), $this->glProductName . '-');
+ $tmpFilePath = $tmpFolder . '.zip';
+ rename($tmpFolder, $tmpFilePath);
+ $url = sprintf(self::$url_translations, $this->glProductName);
+
+ $this->log(sprintf("Downloading $url to $tmpFilePath"));
+
+ $ch = curl_init($url);
+ curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+ curl_setopt($ch, CURLOPT_USERPWD, $this->glUser. ":" . $this->glPassword);
+ $data = curl_exec($ch);
+ $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+ if($code >= 400) {
+ throw new BuildException(sprintf(
+ 'Error downloading %s: %s %s',
+ $url,
+ $code,
+ $data
+ ));
+ }
+ if(curl_error($ch)) {
+ throw new BuildException(sprintf(
+ 'Error downloading %s: %s (#%s)',
+ $url,
+ curl_error($ch),
+ curl_errno($ch)
+ ));
+ }
+
+ curl_close($ch);
+ file_put_contents($tmpFilePath, $data);
+
+ $this->log(sprintf("Extracting to $tmpFolder"));
+ $this->exec("unzip $tmpFilePath -d $tmpFolder");
+
+ return $tmpFolder;
+ }
+
+ /**
+ * @param String Absolute path to a folder structure containing translation files
+ * @return Array with file paths
+ */
+ protected function findFiles($path) {
+ // Recursively find files with certain extensions.
+ // Can't use glob() since its non-recursive.
+ // Directory structure doesn't matter here.
+ $files = array();
+ $matches = new RegexIterator(
+ new RecursiveIteratorIterator(
+ new RecursiveDirectoryIterator($path)
+ ),
+ '/^.+\.(yml$|js)/i',
+ RecursiveRegexIterator::GET_MATCH
+ );
+ foreach($matches as $match) $files[] = $match[0];
+ return $files;
+ }
+
+ protected function processYmlFile($file) {
+ $this->log(sprintf("Processing $file"));
+
+ // Rename locale to correct convention (underscored rather than dashed).
+ // GL wants filenames to adhere to its saved locales, but SS framework
+ // can't easily change to that format for backwards compat reasons, so we need to convert.
+ // The passed in file name doesn't really matter here, only the contained locale.
+ // By convention, the first line in the YAML file is always the locale used, as a YAML "root key".
+ $content = file_get_contents($file);
+ preg_match('/^([\w-_]*):/', $content, $matches);
+ $locale = $matches[1];
+ $locale = str_replace('-', '_', $locale);
+ $locale = str_replace(':', '', $locale);
+
+ // Convert faulty multiline double quoted string YAML
+ // to block format, in order to allow the YAML Parser to open it later
+ // TODO Remove once getlocalization.com has fixed their output format (see support.getlocalization.com #2022)
+ $isBlock = false;
+ $blockIndex = -1;
+ $lines = explode(PHP_EOL, $content);
+ $keyedLineRegex = '/^\s*[\w\d-_]*:\s*/';
+ $leadingQuoteRegex = '/^\s*\"/';
+ $trailingQuoteRegex = '/[^\\\\]\"$/';
+ foreach($lines as $i => $line) {
+ preg_match($keyedLineRegex, $line, $matches);
+ $key = $matches ? $matches[0] : null;
+ $val = trim(preg_replace($keyedLineRegex, '', $line));
+ // If its a multiline double quoted string (no unescaped closing quote)
+ if($val && $line != '"' && preg_match($leadingQuoteRegex, $val) && !preg_match($trailingQuoteRegex, $val)) {
+ $isBlock = true;
+ $blockIndex = $i;
+ } elseif($key) {
+ $isBlock = false;
+ $blockIndex = -1;
+ } else {
+ $lines[$blockIndex] .= $line;
+ unset($lines[$i]);
+ }
+ }
+ $content = implode(PHP_EOL, $lines);
+
+ // Parse YML as a sanity check,
+ // and reorder alphabetically by key to ensure consistent diffs.
+ require_once '../framework/thirdparty/zend_translate_railsyaml/library/Translate/Adapter/thirdparty/sfYaml/lib/sfYaml.php';
+ require_once '../framework/thirdparty/zend_translate_railsyaml/library/Translate/Adapter/thirdparty/sfYaml/lib/sfYamlParser.php';
+ require_once '../framework/thirdparty/zend_translate_railsyaml/library/Translate/Adapter/thirdparty/sfYaml/lib/sfYamlDumper.php';
+ $yamlHandler = new sfYaml();
+ $yml = $yamlHandler->parse($content);
+ if(isset($yml[$locale]) && is_array($yml[$locale])) {
+ ksort($yml[$locale]);
+ foreach($yml[$locale] as $k => &$v) {
+ if(is_array($v)) ksort($v);
+ }
+ }
+ $content = $yamlHandler->dump($yml, 99); // don't inline first levels
+
+ // Save into correct path, overwriting existing files
+ $path = $this->modulePath . '/' . $this->langFolders['yml'] . '/' . $locale . '.yml';
+ $this->log("Saving to $path");
+ file_put_contents($path, $content);
+ }
+
+ protected function processJavascriptFile($file) {
+ $this->log(sprintf("Processing $file"));
+
+ $locale = pathinfo($file, PATHINFO_FILENAME);
+ $locale = str_replace('-', '_', $locale);
+
+ // Save into correct path, overwriting existing files
+ $path = $this->modulePath . '/' . $this->langFolders['js'] . '/' . $locale . '.yml';
+ $this->log("Saving to $path");
+ file_put_contents($path, file_get_contents($file));
+ }
+}
+
+
+?>

0 comments on commit 00f66e2

Please sign in to comment.
Something went wrong with that request. Please try again.