From efbe24e1bfc7e45392c31d75e60c115d98109b5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Tue, 12 Jun 2018 11:15:56 +0200 Subject: [PATCH 1/6] add travis_makepot --- travis/test_server.py | 29 +++++++++++++++++++++++++++++ travis/travis_makepot | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100755 travis/travis_makepot diff --git a/travis/test_server.py b/travis/test_server.py index 8301d1460..567445ee0 100755 --- a/travis/test_server.py +++ b/travis/test_server.py @@ -11,6 +11,20 @@ get_addons, get_modules, get_modules_info, get_dependencies) from travis_helpers import success_msg, fail_msg from configparser import ConfigParser +from contextlib import contextmanager + + +@contextmanager +def setenv(key, value): + old_value = os.environ.get(key) + os.environ[key] = value + try: + yield + finally: + if old_value: + os.environ[key] = old_value + else: + del os.environ[key] def has_test_errors(fname, dbname, odoo_version, check_loaded=True): @@ -463,6 +477,21 @@ def main(argv=None): return 1 elif counted_errors != expected_errors: return 1 + # no test error, let's generate .pot and msgmerge all .po files + must_run_makepot = ( + os.environ.get('MAKEPOT') == '1' and + os.environ.get('TRAVIS_PULL_REQUEST') == 'false' and + os.environ.get('GITHUB_USER') and + os.environ.get('GITHUB_TOKEN') + ) + if must_run_makepot: + with setenv('PYTHONPATH', server_path): + makepot_cmd = ['unbuffer'] if unbuffer else [] + makepot_cmd += [ + 'travis_makepot', + ] + if subprocess.call(makepot_cmd) != 0: + return 1 # if we get here, all is OK return 0 diff --git a/travis/travis_makepot b/travis/travis_makepot new file mode 100755 index 000000000..3e6fcbf66 --- /dev/null +++ b/travis/travis_makepot @@ -0,0 +1,40 @@ +#!/usr/bin/env python + +from __future__ import print_function +import os +import subprocess + + +def main(): + database = os.environ.get('MQT_TEST_DB', 'openerp_test') + print("Installing click-odoo-contrib") + # TODO release to pypi click-odoo-contrib makepot + r = subprocess.call([ + 'pip', 'install', + 'git+https://github.com/acsone/click-odoo-contrib@master', + ]) + if r: + return r + print("Exporting .pot files and updating .po files") + r = subprocess.call([ + 'click-odoo-makepot', + '-d', database, + '--commit', + '--log-level=debug', + ]) + if r: + return r + print("Pushing .pot and .po files changes to GitHub") + r = subprocess.call([ + 'git', 'push', + 'https://{GITHUB_USER}:{GITHUB_TOKEN}@github.com/' + '{TRAVIS_REPO_SLUG}'.format(**os.environ), + 'HEAD:{TRAVIS_BRANCH}'.format(**os.environ), + ]) + if r: + return r + return 0 + + +if __name__ == "__main__": + exit(main()) From 20a6cf183a801c55591d0152c4c4ace2b2bb307b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Wed, 13 Jun 2018 15:12:33 +0200 Subject: [PATCH 2/6] run makepot if TRANSIFEX=1 In this case, run it --- travis/test_server.py | 29 ++++--------- travis/travis_makepot | 40 ------------------ travis/travis_makepot.py | 88 ++++++++++++++++++++++++++++++++++++++++ travis/travis_run_tests | 15 +++++++ 4 files changed, 111 insertions(+), 61 deletions(-) delete mode 100755 travis/travis_makepot create mode 100755 travis/travis_makepot.py diff --git a/travis/test_server.py b/travis/test_server.py index 567445ee0..841822885 100755 --- a/travis/test_server.py +++ b/travis/test_server.py @@ -11,20 +11,6 @@ get_addons, get_modules, get_modules_info, get_dependencies) from travis_helpers import success_msg, fail_msg from configparser import ConfigParser -from contextlib import contextmanager - - -@contextmanager -def setenv(key, value): - old_value = os.environ.get(key) - os.environ[key] = value - try: - yield - finally: - if old_value: - os.environ[key] = old_value - else: - del os.environ[key] def has_test_errors(fname, dbname, odoo_version, check_loaded=True): @@ -485,13 +471,14 @@ def main(argv=None): os.environ.get('GITHUB_TOKEN') ) if must_run_makepot: - with setenv('PYTHONPATH', server_path): - makepot_cmd = ['unbuffer'] if unbuffer else [] - makepot_cmd += [ - 'travis_makepot', - ] - if subprocess.call(makepot_cmd) != 0: - return 1 + # run makepot using the database we just tested + makepot_cmd = ['unbuffer'] if unbuffer else [] + makepot_cmd += [ + 'travis_makepot', + database, + ] + if subprocess.call(makepot_cmd) != 0: + return 1 # if we get here, all is OK return 0 diff --git a/travis/travis_makepot b/travis/travis_makepot deleted file mode 100755 index 3e6fcbf66..000000000 --- a/travis/travis_makepot +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -import os -import subprocess - - -def main(): - database = os.environ.get('MQT_TEST_DB', 'openerp_test') - print("Installing click-odoo-contrib") - # TODO release to pypi click-odoo-contrib makepot - r = subprocess.call([ - 'pip', 'install', - 'git+https://github.com/acsone/click-odoo-contrib@master', - ]) - if r: - return r - print("Exporting .pot files and updating .po files") - r = subprocess.call([ - 'click-odoo-makepot', - '-d', database, - '--commit', - '--log-level=debug', - ]) - if r: - return r - print("Pushing .pot and .po files changes to GitHub") - r = subprocess.call([ - 'git', 'push', - 'https://{GITHUB_USER}:{GITHUB_TOKEN}@github.com/' - '{TRAVIS_REPO_SLUG}'.format(**os.environ), - 'HEAD:{TRAVIS_BRANCH}'.format(**os.environ), - ]) - if r: - return r - return 0 - - -if __name__ == "__main__": - exit(main()) diff --git a/travis/travis_makepot.py b/travis/travis_makepot.py new file mode 100755 index 000000000..a2b1efcc1 --- /dev/null +++ b/travis/travis_makepot.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +from __future__ import print_function +from contextlib import contextmanager +import os +import subprocess + +from test_server import setup_server, get_addons_path, \ + get_server_path, get_addons_to_check, create_server_conf, get_server_script + + +@contextmanager +def setenv(key, value): + old_value = os.environ.get(key) + os.environ[key] = value + try: + yield + finally: + if old_value: + os.environ[key] = old_value + else: + del os.environ[key] + + +def main(database=None): + odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo") + odoo_version = os.environ.get("VERSION") + odoo_exclude = os.environ.get("EXCLUDE") + odoo_include = os.environ.get("INCLUDE") + odoo_unittest = False + install_options = os.environ.get("INSTALL_OPTIONS", "").split() + travis_home = os.environ.get("HOME", "~/") + travis_dependencies_dir = os.path.join(travis_home, 'dependencies') + travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..") + server_path = get_server_path(odoo_full, odoo_version, travis_home) + addons_path = get_addons_path(travis_dependencies_dir, + travis_build_dir, + server_path) + addons_list = get_addons_to_check(travis_build_dir, odoo_include, + odoo_exclude) + addons = ','.join(addons_list) + create_server_conf({'addons_path': addons_path}, odoo_version) + + print("\nWorking in %s" % travis_build_dir) + print("Using repo %s and addons path %s" % (odoo_full, addons_path)) + + if not database: + database = 'openerp_i18n' + print("\nInitializing db %s to generate .pot for: %s" % (database, addons)) + script_name = get_server_script(server_path) + setup_server(database, odoo_unittest, addons_list, server_path, + script_name, addons_path, install_options, addons_list) + else: + print("\nUsing db %s to generate .pot" % (database, )) + + print("Installing click-odoo-contrib") + r = subprocess.call([ + 'pip', 'install', 'click-odoo-contrib', + ]) + if r: + return r + + with setenv('PYTHONPATH', server_path): + print("Exporting .pot files") + r = subprocess.call([ + 'click-odoo-makepot', + '-d', database, + '--commit', + '--log-level=debug', + ]) + if r: + return r + + print("Pushing .pot files changes to GitHub") + r = subprocess.call([ + 'git', 'push', + 'https://{GITHUB_USER}:{GITHUB_TOKEN}@github.com/' + '{TRAVIS_REPO_SLUG}'.format(**os.environ), + 'HEAD:{TRAVIS_BRANCH}'.format(**os.environ), + ]) + if r: + return r + + return 0 + + +if __name__ == "__main__": + exit(main(sys.argv[1])) diff --git a/travis/travis_run_tests b/travis/travis_run_tests index 53473519b..adde98a28 100755 --- a/travis/travis_run_tests +++ b/travis/travis_run_tests @@ -63,6 +63,18 @@ if __name__ == '__main__': # Avoid false red from forks using OCA transifex user is_valid_transifex = not is_pull_request and \ (is_oca_project or not is_oca_transifex_user) + # run makepot using a fresh database + # where addons are going to be installed + # is MAKEPOT=1 and TESTS=1, test_server.py will run + # makepot using the test database, which will be faster + must_run_makepot = ( + (os.environ.get('MAKEPOT') == '1' or + os.environ.get('TRANSIFEX') == '1') and + not tests_enabled and + not is_pull_request and + os.environ.get('GITHUB_USER') and + os.environ.get('GITHUB_TOKEN') + ) # Test list. Each test is a list with command + arguments. tests = [] @@ -80,6 +92,9 @@ if __name__ == '__main__': if transifex_enabled and is_valid_transifex: tests.append(['travis_transifex.py']) + if must_run_makepot: + tests.append(['travis_makepot.py']) + is_valid_weblate = ( not is_pull_request and os.environ.get('UNIT_TEST') != '1' and ['test_server.py'] in tests) From 96e48181d6494334419c0f318364c8aab48e9005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Wed, 13 Jun 2018 15:19:37 +0200 Subject: [PATCH 3/6] remove transifex support --- travis/travis_run_tests | 9 -- travis/travis_transifex.py | 192 ------------------------------------- 2 files changed, 201 deletions(-) delete mode 100755 travis/travis_transifex.py diff --git a/travis/travis_run_tests b/travis/travis_run_tests index adde98a28..a8c747a04 100755 --- a/travis/travis_run_tests +++ b/travis/travis_run_tests @@ -52,17 +52,11 @@ if __name__ == '__main__': lint_check_enabled = os.environ.get('LINT_CHECK') == '1' tests_enabled = os.environ.get('TESTS') == '1' tests_unspecified = os.environ.get('TESTS') is None - transifex_enabled = os.environ.get('TRANSIFEX') == '1' weblate_enabled = os.environ.get('WEBLATE') == '1' is_oca_project = os.environ.get('TRAVIS_REPO_SLUG', '').startswith('OCA/') - is_oca_transifex_user = os.environ.get('TRANSIFEX_USER') == \ - 'transbot@odoo-community.org' # TRAVIS_PULL_REQUEST contains the pull request number or 'false' is_pull_request = os.environ.get('TRAVIS_PULL_REQUEST') != 'false' - # Avoid false red from forks using OCA transifex user - is_valid_transifex = not is_pull_request and \ - (is_oca_project or not is_oca_transifex_user) # run makepot using a fresh database # where addons are going to be installed # is MAKEPOT=1 and TESTS=1, test_server.py will run @@ -89,9 +83,6 @@ if __name__ == '__main__': elif tests_enabled: tests.append(['test_server.py']) - if transifex_enabled and is_valid_transifex: - tests.append(['travis_transifex.py']) - if must_run_makepot: tests.append(['travis_makepot.py']) diff --git a/travis/travis_transifex.py b/travis/travis_transifex.py deleted file mode 100755 index bce2bcfef..000000000 --- a/travis/travis_transifex.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from __future__ import unicode_literals -import os -import sys -import time -import subprocess -from slumber import API, exceptions -from odoo_connection import context_mapping, Odoo10Context -from test_server import setup_server, get_addons_path, \ - get_server_path, get_addons_to_check, create_server_conf, get_server_script -from travis_helpers import yellow, yellow_light, red -from txclib import utils, commands - - -def main(argv=None): - """ - Export translation files and push them to Transifex - The transifex password should be encrypted in .travis.yml - If not, export exits early. - """ - if argv is None: - argv = sys.argv - - transifex_user = os.environ.get("TRANSIFEX_USER") - transifex_password = os.environ.get("TRANSIFEX_PASSWORD") - - if not transifex_user: - print(yellow_light("WARNING! Transifex user not defined- " - "exiting early.")) - return 1 - - if not transifex_password: - print(yellow_light("WARNING! Transifex password not recognized- " - "exiting early.")) - return 1 - - travis_home = os.environ.get("HOME", "~/") - travis_dependencies_dir = os.path.join(travis_home, 'dependencies') - travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..") - travis_repo_slug = os.environ.get("TRAVIS_REPO_SLUG") - travis_repo_owner = travis_repo_slug.split("/")[0] - travis_repo_shortname = travis_repo_slug.split("/")[1] - odoo_unittest = False - odoo_exclude = os.environ.get("EXCLUDE") - odoo_include = os.environ.get("INCLUDE") - install_options = os.environ.get("INSTALL_OPTIONS", "").split() - odoo_version = os.environ.get("VERSION") - - if not odoo_version: - # For backward compatibility, take version from parameter - # if it's not globally set - odoo_version = argv[1] - print(yellow_light("WARNING: no env variable set for VERSION. " - "Using '%s'" % odoo_version)) - - default_project_slug = "%s-%s" % (travis_repo_slug.replace('/', '-'), - odoo_version.replace('.', '-')) - transifex_project_slug = os.environ.get("TRANSIFEX_PROJECT_SLUG", - default_project_slug) - transifex_project_name = "%s (%s)" % (travis_repo_shortname, odoo_version) - transifex_organization = os.environ.get("TRANSIFEX_ORGANIZATION", - travis_repo_owner) - transifex_fill_up_resources = os.environ.get( - "TRANSIFEX_FILL_UP_RESOURCES", "True" - ) - transifex_team = os.environ.get( - "TRANSIFEX_TEAM", "23907" - ) - repository_url = "https://github.com/%s" % travis_repo_slug - - odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo") - server_path = get_server_path(odoo_full, odoo_version, travis_home) - addons_path = get_addons_path(travis_dependencies_dir, - travis_build_dir, - server_path) - addons_list = get_addons_to_check(travis_build_dir, odoo_include, - odoo_exclude) - addons = ','.join(addons_list) - create_server_conf({'addons_path': addons_path}, odoo_version) - - print("\nWorking in %s" % travis_build_dir) - print("Using repo %s and addons path %s" % (odoo_full, addons_path)) - - if not addons: - print(yellow_light("WARNING! Nothing to translate- exiting early.")) - return 0 - - # Create Transifex project if it doesn't exist - print() - print(yellow("Creating Transifex project if it doesn't exist")) - auth = (transifex_user, transifex_password) - api_url = "https://www.transifex.com/api/2/" - api = API(api_url, auth=auth) - project_data = {"slug": transifex_project_slug, - "name": transifex_project_name, - "source_language_code": "en", - "description": transifex_project_name, - "repository_url": repository_url, - "organization": transifex_organization, - "license": "permissive_open_source", - "fill_up_resources": transifex_fill_up_resources, - "team": transifex_team, - } - try: - api.project(transifex_project_slug).get() - print('This Transifex project already exists.') - except exceptions.HttpClientError: - try: - api.projects.post(project_data) - print('Transifex project has been successfully created.') - except exceptions.HttpClientError: - print('Transifex organization: %s' % transifex_organization) - print('Transifex username: %s' % transifex_user) - print('Transifex project slug: %s' % transifex_project_slug) - print(red('Error: Authentication failed. Please verify that ' - 'Transifex organization, user and password are ' - 'correct. You can change these variables in your ' - '.travis.yml file.')) - raise - - print("\nModules to translate: %s" % addons) - - # Install the modules on the database - database = "openerp_i18n" - script_name = get_server_script(server_path) - setup_server(database, odoo_unittest, addons_list, server_path, - script_name, addons_path, install_options, addons_list) - - # Initialize Transifex project - print() - print(yellow('Initializing Transifex project')) - init_args = ['--host=https://www.transifex.com', - '--user=%s' % transifex_user, - '--pass=%s' % transifex_password] - commands.cmd_init(init_args, path_to_tx=None) - path_to_tx = utils.find_dot_tx() - - # Use by default version 10 connection context - connection_context = context_mapping.get(odoo_version, Odoo10Context) - with connection_context(server_path, addons_path, database) \ - as odoo_context: - for module in addons_list: - print() - print(yellow("Obtaining POT file for %s" % module)) - i18n_folder = os.path.join(travis_build_dir, module, 'i18n') - source_filename = os.path.join(i18n_folder, module + ".pot") - # Create i18n/ directory if doesn't exist - if not os.path.exists(os.path.dirname(source_filename)): - os.makedirs(os.path.dirname(source_filename)) - pot_contents = odoo_context.get_pot_contents(module) - if isinstance(pot_contents, str): - f = open(source_filename, 'w') - else: # Odoo in Python 3 returns bytes - f = open(source_filename, 'wb') - f.write(pot_contents) - f.close() - # Put the correct timestamp for letting known tx client which - # translations to update - for po_file_name in os.listdir(i18n_folder): - if not po_file_name.endswith('.po'): - continue - po_file_name = os.path.join(i18n_folder, po_file_name) - command = ['git', 'log', '--pretty=format:%cd', '-n1', - '--date=raw', po_file_name] - timestamp = float(subprocess.check_output(command).split()[0]) - # This converts to UTC the timestamp - timestamp = time.mktime(time.gmtime(timestamp)) - os.utime(po_file_name, (timestamp, timestamp)) - - print() - print(yellow("Linking POT file and Transifex resource")) - set_args = ['-t', 'PO', - '--auto-local', - '-r', '%s.%s' % (transifex_project_slug, module), - '%s/i18n/.po' % module, - '--source-lang', 'en', - '--source-file', source_filename, - '--execute'] - commands.cmd_set(set_args, path_to_tx) - - print() - print(yellow('Pushing translation files to Transifex')) - push_args = ['-s', '-t', '--skip'] - commands.cmd_push(push_args, path_to_tx) - - return 0 - - -if __name__ == "__main__": - exit(main()) From 1535d2baa0e3f72f12c2902bb95d0da194996e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Wed, 13 Jun 2018 15:22:07 +0200 Subject: [PATCH 4/6] remove weblate support --- travis/apis.py | 124 ------------------ travis/travis_run_tests | 9 -- travis/travis_weblate.py | 265 --------------------------------------- 3 files changed, 398 deletions(-) delete mode 100755 travis/travis_weblate.py diff --git a/travis/apis.py b/travis/apis.py index 44024658d..9d99c0ecf 100644 --- a/travis/apis.py +++ b/travis/apis.py @@ -2,12 +2,7 @@ import base64 import os -import re -import subprocess -import tempfile -import time import json -from contextlib import contextmanager import requests @@ -44,125 +39,6 @@ def _request(self, url, payload=None, is_json=True, patch=False): return response.json() if is_json else response -class WeblateApi(Request): - - def __init__(self): - super(WeblateApi, self).__init__() - self.repo_slug = None - self.branch = None - self._token = os.environ.get("WEBLATE_TOKEN") - self.host = os.environ.get( - "WEBLATE_HOST", "https://weblate.odoo-community.org/api") - self.ssh = os.environ.get( - "WEBLATE_SSH", "ssh://user@webpage.com") - self.tempdir = os.path.join(tempfile.gettempdir(), 'weblate_api') - self._ssh_keyscan() - - def _ssh_keyscan(self): - """This method execute the command 'ssh-keysan' to avoid the - question when the command git clone is excecuted. - The question is like to: - 'Are you sure you want to continue connecting (yes/no)?'""" - cmd = ['ssh-keyscan', '-p'] - match = re.search( - r'(ssh\:\/\/\w+@(?P[a-zA-Z0-9_.-]+))(:{0,1})' - r'(?P(\d+))?', self.ssh) - if not match: - return False - data = match.groupdict() - cmd.append(data['port'] or '22') - cmd.append(data['host']) - with open(os.path.expanduser('~/.ssh/known_hosts'), 'a+') as hosts: - subprocess.Popen(cmd, stdout=hosts) - - def get_project(self, repo_slug, branch): - self.branch = branch - projects = [] - page = 1 - while True: - data = self._request(self.host + '/projects/?page=%s' % page) - projects.extend(data['results'] or []) - if not data['next']: - break - page += 1 - for project in projects: - if project['name'] == repo_slug: - self.repo_slug = project['slug'] - return project - raise ApiException('No project found in "%s" for this path "%s"' % - (self.host, repo_slug)) - - def load_project(self, repo_slug, branch): - self.project = self.get_project(repo_slug, branch) - self.load_components() - - def get_components(self): - components = [] - values = [] - page = 1 - while True: - data = self._request(self.host + - '/projects/%s/components/?page=%s' % - (self.project['slug'], page)) - values.extend(data['results'] or []) - if not data['next']: - break - page += 1 - for value in values: - if value['branch'] and value['branch'] != self.branch: - continue - components.append(value) - return components - - def load_components(self): - self.components = self.get_components() - - def pull(self): - pull = self._request( - self.host + '/projects/%s/repository/' % self.project['slug'], - {'operation': 'pull'}) - return pull['result'] - - def component_repository(self, component, operation): - result = self._request(self.host + '/components/%s/%s/repository/' % - (self.project['slug'], component['slug']), - {'operation': operation}) - return result['result'] - - @contextmanager - def component_lock(self): - try: - for component in self.components: - self._component_lock(component) - self._component_commit(component) - yield - finally: - for component in self.components: - self._component_lock(component, lock=False) - - def _component_lock(self, component, lock=True): - url = (self.host + '/components/%s/%s/lock/' % - (self.project['slug'], component['slug'])) - for i in range(10): - new_lock = self._request(url, {'lock': lock}) - if new_lock['locked'] == lock: - break - time.sleep(60) - return True - - def _component_commit(self, component): - url = (self.host + '/components/%s/%s/repository/' % - (self.project['slug'], component['slug'])) - needs_commit = self._request(url) - if not needs_commit['needs_commit']: - return - new_commit = self._request(url, {'operation': 'commit'}) - if not new_commit['result']: - raise ApiException('The commit into the component "%s" cannot be ' - 'made' % component['slug']) - return True - - class GitHubApi(Request): def __init__(self): diff --git a/travis/travis_run_tests b/travis/travis_run_tests index a8c747a04..472dd65fb 100755 --- a/travis/travis_run_tests +++ b/travis/travis_run_tests @@ -52,7 +52,6 @@ if __name__ == '__main__': lint_check_enabled = os.environ.get('LINT_CHECK') == '1' tests_enabled = os.environ.get('TESTS') == '1' tests_unspecified = os.environ.get('TESTS') is None - weblate_enabled = os.environ.get('WEBLATE') == '1' is_oca_project = os.environ.get('TRAVIS_REPO_SLUG', '').startswith('OCA/') # TRAVIS_PULL_REQUEST contains the pull request number or 'false' @@ -86,13 +85,5 @@ if __name__ == '__main__': if must_run_makepot: tests.append(['travis_makepot.py']) - is_valid_weblate = ( - not is_pull_request and os.environ.get('UNIT_TEST') != '1' and - ['test_server.py'] in tests) - - if weblate_enabled and is_valid_weblate: - # Re-use database of TESTS=1 except if UNIT_TEST is enabled. - tests.append(['travis_weblate.py']) - if tests: exit(main(tests)) diff --git a/travis/travis_weblate.py b/travis/travis_weblate.py deleted file mode 100755 index dfadc7b28..000000000 --- a/travis/travis_weblate.py +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os -import re -import glob -import subprocess - -from odoo_connection import Odoo10Context, context_mapping -from test_server import (get_addons_path, get_server_path, parse_list) -from apis import ApiException, WeblateApi, GitHubApi -from travis_helpers import yellow -from git_run import GitRun - - -class TravisWeblateUpdate(object): - - GIT_COMMIT_INFO = { - 'author': 'Weblate bot ', - 'message': '[REF] i18n: Updating translation terms from weblate ' - '[ci skip]' - } - - def __init__(self): - self._git = GitRun(os.path.join(os.getcwd(), '.git'), True) - self.branch = os.environ.get("TRAVIS_BRANCH", - self._git.get_branch_name()) - remote = self._git.run(["ls-remote", "--get-url", "origin"]) - name = remote.replace(':', '/') - name = re.sub('.+@', '', name) - name = re.sub('.git$', '', name) - name = re.sub('^https://', '', name) - name = re.sub('^http://', '', name) - match = re.search( - r'(?P[^/]+)/(?P[^/]+)/(?P[^/]+)', name) - if match: - name = ("%(host)s:%(owner)s/%(repo)s (%(branch)s)" % - dict(match.groupdict(), branch=self.branch)) - self.repo_name = name - self.wl_api = WeblateApi() - self.gh_api = GitHubApi() - self._travis_home = os.environ.get("HOME", "~/") - self._travis_build_dir = os.environ.get("TRAVIS_BUILD_DIR", "../..") - self._odoo_version = os.environ.get("VERSION") - self._odoo_branch = os.environ.get("ODOO_BRANCH") - self._langs = (parse_list(os.environ.get("LANG_ALLOWED")) if - os.environ.get("LANG_ALLOWED", False) else []) - self._odoo_full = os.environ.get("ODOO_REPO", "odoo/odoo") - self._server_path = get_server_path(self._odoo_full, - (self._odoo_branch or - self._odoo_version), - self._travis_home) - self._addons_path = get_addons_path(self._travis_home, - self._travis_build_dir, - self._server_path) - self._database = os.environ.get('MQT_TEST_DB', 'openerp_test') - self._connection_context = context_mapping.get( - self._odoo_version, Odoo10Context) - self._apply_patch_odoo() - self._get_modules_installed() - - def _check(self): - self.wl_api._check() - self.gh_api._check() - - def _apply_patch_odoo(self): - """This patch is necessary because the weblate does not check which - word and the translated are the same to use it in its percentage of - translated""" - for base in ('odoo', 'openerp'): - p_file = os.path.join(self._server_path, base, - os.path.join('tools', 'translate.py')) - if os.path.isfile(p_file): - sed = ["sed", "-i", "-e", - r"s/translation'] = src/translation'] = ''/g", p_file] - print(" ".join(sed)) - subprocess.call(sed) - p_file = os.path.join(self._server_path, base, - os.path.join('addons', 'base', 'ir', - 'ir_translation.py')) - if os.path.isfile(p_file): - sed = ["sed", "-i", "-e", - r"s/if\snot\strans_dict\['value']:/if False:/g", p_file] - print(" ".join(sed)) - subprocess.call(sed) - - def _get_modules_installed(self): - self._installed_modules = [] - modules_found = [] - for name in ('__openerp__.py', '__manifest__.py'): - modules = glob.glob('%s/**/%s' % (self._travis_build_dir, name)) - if not modules: - continue - modules_found.extend([ - os.path.dirname(module).split('/')[-1] for module in - modules]) - with self._connection_context(self._server_path, self._addons_path, - self._database) as odoo_context: - odoo_context.cr.execute("select name from ir_module_module" - " where state = 'installed' and " - "name in %s", (tuple(modules_found),)) - modules = odoo_context.cr.dictfetchall() - self._installed_modules = [module['name'] for module in modules] - - def _generate_odoo_po_files(self, module, only_installed=True): - generated = False - with self._connection_context(self._server_path, self._addons_path, - self._database) as odoo_context: - if only_installed and module not in self._installed_modules: - return generated - print("\n", yellow("Obtaining POT file for %s" % module)) - i18n_folder = os.path.join(self._travis_build_dir, module, 'i18n') - if not os.path.isdir(i18n_folder): - os.makedirs(i18n_folder) - # Put git add for letting known git which translations to update - po_files = glob.glob(os.path.join(i18n_folder, '*.po')) - for lang in self._langs: - if os.path.isfile(os.path.join(i18n_folder, lang + '.po')): - continue - po_content = odoo_context.get_pot_contents(module, lang) - if not po_content: - continue - with open(os.path.join(i18n_folder, lang + '.po'), 'wb')\ - as f_po: - f_po.write(po_content) - if self._git.run(["add", "-v", f_po.name]): - generated = True - for po_file_name in po_files: - lang = os.path.basename(os.path.splitext(po_file_name)[0]) - if self._langs and lang not in self._langs: - # Limit to only allowed languages if it's defined - continue - po_file_path = os.path.join(i18n_folder, po_file_name) - with open(po_file_path, 'r') as f_po: - odoo_context.load_po(f_po, lang) - new_content = odoo_context.get_pot_contents(module, lang) - if not new_content: - continue - with open(po_file_path, 'wb') as f_po: - f_po.write(new_content) - diff = self._git.run(["diff", "HEAD", po_file_path]) - if diff.count('msgstr') == 1: - self._git.run(["checkout", po_file_path]) - if self._git.run(["add", "-v"] + po_files): - generated = True - return generated - - def _check_conflict(self, component): - status = self._git.run(["status"]) - conflicts = [item for item in status.split('\n') - if (item.startswith('\tboth modified') and - component['filemask'].replace('/*.po', '') in item)] - if conflicts: - self._register_pull_request(component, status) - return True - return False - - def _register_pull_request(self, component, status): - branch_name = 'conflict-%s-weblate' % self.branch - self._git.run(["add", component['filemask']]) - self._git.run(["commit", "--no-verify", - "--author='Weblate bot '", - "-m", "[REF] i18n: Conflict on the daily cron", - "-m", status]) - self._git.run(["branch", "-m", branch_name]) - self._git.run(["push", "-f", "origin", branch_name]) - pull = self.gh_api.create_pull_request({ - 'title': '[REF] i18n: Conflict on the daily cron', - 'head': '%s:%s' % (self.repo_name.split('/')[0].split(':')[1], - branch_name), - 'base': self.branch, - 'body': status - }) - self._git.run(["checkout", "-qb", self.branch, - "origin/%s" % self.branch]) - self._git.run(["branch", "-D", branch_name]) - print(yellow("The pull request register is: %s" % pull['html_url'])) - - def _commit_weblate(self, first_commit=False): - if ('nothing to commit, working tree clean' - in self._git.run(["status"])): - return first_commit - if first_commit: - self._git.run(["commit", "--no-verify", "--amend", - "--no-edit"]) - else: - self._git.run(["commit", "--no-verify", - "--author='%s'" % self.GIT_COMMIT_INFO['author'], - "-m", self.GIT_COMMIT_INFO['message']]) - first_commit = True - return first_commit - - def _push_git_repository(self): - po_files = self._git.run(["show", "--format=format:'%H'", - "--name-only"]).split('\n') - if not len(po_files) > 1: - return False - commit = self.gh_api.create_commit(self.GIT_COMMIT_INFO['message'], - self.branch, - po_files[1:]) - if commit: - for component in self.wl_api.components: - self.wl_api.component_repository(component, 'reset') - self.wl_api.component_repository(component, 'pull') - return commit - - def update(self): - self._check() - self.wl_api.load_project(self.repo_name, self.branch) - with self.wl_api.component_lock(): - self._git.run(["fetch", "origin"]) - first_commit = False - component = [item for item in self.wl_api.components - if item['git_export']] - if len(component) > 1: - print(yellow("To many repository for this project %s" % - self.wl_api.project['name'])) - return 1 - remote = (self.wl_api.ssh + '/' + self.wl_api.project['slug'] + - '/' + component[0]['slug']) - name = '%s-wl' % self.wl_api.project['slug'] - self._git.run(["remote", "add", name, remote]) - for component in self.wl_api.components: - print(yellow("Component %s" % component['slug'])) - self._git.run(["checkout", "-qb", self.branch, - "origin/%s" % self.branch]) - self.wl_api.component_repository(component, 'pull') - self._git.run(["fetch", name]) - if self._generate_odoo_po_files(component['name']): - first_commit = self._commit_weblate(first_commit) - self._git.run(["merge", "--squash", "-s", "recursive", "-X", - "ours", "%s/%s" % (name, self.branch)]) - if self._check_conflict(component): - break - if (component['filemask'].replace('/*.po', '') in - self._git.run(["status"])): - self._git.run(["add", component['filemask']]) - first_commit = self._commit_weblate(first_commit) - if self._check_conflict(component): - break - first_commit = self._commit_weblate(first_commit) - self._git.run(["remote", "remove", name]) - modules_no_processed = [module for module in - self._installed_modules if module not in - [comp['name'] for comp in - self.wl_api.components]] - for component in modules_no_processed: - if self._generate_odoo_po_files(component, - only_installed=False): - first_commit = self._commit_weblate(first_commit) - if not self._push_git_repository(): - return 1 - return 0 - - -def main(argv=None): - try: - TravisWeblateUpdate().update() - except ApiException as exc: - print(yellow(str(exc))) - raise exc - - -if __name__ == "__main__": - main() From d35cd7120b81c70c61c1651b595ec0bb538f35f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Wed, 13 Jun 2018 15:37:59 +0200 Subject: [PATCH 5/6] remove transifex support --- sample_files/README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sample_files/README.md b/sample_files/README.md index ee53de080..b068405d9 100644 --- a/sample_files/README.md +++ b/sample_files/README.md @@ -10,10 +10,6 @@ ${REPO_DESCRIPTION} This part will be replaced when running the oca-gen-addons-table script from OCA/maintainer-tools. [//]: # (end addons) -Translation Status ------------------- -[![Transifex Status](https://www.transifex.com/projects/p/${ORG_NAME}-${REPO_NAME}-${BRANCH_NAME_WITH_DASH}/chart/image_png)](https://www.transifex.com/projects/p/${ORG_NAME}-${REPO_NAME}-${BRANCH_NAME_WITH_DASH}) - ---- OCA, or the [Odoo Community Association](http://odoo-community.org/), is a nonprofit organization whose From 86cf7c8e38e86f9403793aa13d56c4098b2c6850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Wed, 13 Jun 2018 15:38:26 +0200 Subject: [PATCH 6/6] update sample .travis.yml --- sample_files/.travis.yml | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/sample_files/.travis.yml b/sample_files/.travis.yml index d04bf8511..5bf19fea0 100644 --- a/sample_files/.travis.yml +++ b/sample_files/.travis.yml @@ -31,7 +31,7 @@ addons: env: global: - - VERSION="10.0" TESTS="0" LINT_CHECK="0" TRANSIFEX="0" + - VERSION="10.0" TESTS="0" LINT_CHECK="0" MAKEPOT="0" # Set this variable to some version existing as linux-generic build on # https://github.com/wkhtmltopdf/wkhtmltopdf/releases # if you need to install wkhtmltopdf @@ -46,28 +46,14 @@ env: # Use the above line to install dependencies that are required for website repos: # * SASS & Bootstrap-SASS # * Compass - - TRANSIFEX_USER='transbot@odoo-community.org' - # This line contains the encrypted transifex password - # To encrypt transifex password, install travis ruby utils with: - # $ gem install travis --user-install - # and use: - # $ travis encrypt TRANSIFEX_PASSWORD=your-password -r owner/project - # Secure list for current OCA projects is in https://github.com/OCA/maintainer-quality-tools/issues/194 - - secure: PjP88tPSwimBv4tsgn3UcQAD1heK/wcuSaSfhi2xUt/jSrOaTmWzjaW2gH/eKM1ilxPXwlPGyAIShJ2JJdBiA97hQufOeiqxdkWDctnNVVEDx2Tk0BiG3PPYyhXPgUZ+FNOnjZFF3pNWvzXTQaB0Nvz8plqp93Ov/DEyhrCxHDs= - # Use the following lines if you need to manually change the transifex project slug or/and the transifex organization. - # The default project slug is owner-repo_name-version (with dash in the version string). - # The default organization is the owner of the repo. - # The default fill up resources (TM) is True. - # The default team is 23907. https://www.transifex.com/organization/oca/team/23907/ - # - TRANSIFEX_PROJECT_SLUG= - # - TRANSIFEX_ORGANIZATION= - # - TRANSIFEX_FILL_UP_RESOURCES= - # - TRANSIFEX_TEAM= matrix: - LINT_CHECK="1" - - TRANSIFEX="1" - - TESTS="1" ODOO_REPO="odoo/odoo" +# use this to install a standalone database to export .pot files +# - MAKEPOT="1" +# add MAKEPOT="1" to a TEST line to export .pot files from +# the test database after test success + - TESTS="1" ODOO_REPO="odoo/odoo" MAKEPOT="1" - TESTS="1" ODOO_REPO="OCA/OCB" # either use the two lines above or the two below. Don't change the default if # it's not necessary (it is only necessary if modules in your repository can't