Improved installation (#56)

* Wrapper for apertium-tagger

* Merge branch 'master' into swig_wrapper_tagger

* Type Fix

* added: os based installation

support for ubuntu & windows

* Fixed: logging

* Fix: coverage

* change: import location

* added: py.typed PEP 561

* Removed: distro

* added: installation of wrapper

* Build only the wrapper

* coverage ignores in .coveragerc

* ubuntu installation

switch to urllib3 from wget

* use existing import statement

* pass list instead of string to subprocess.run()

* dont build lttoolbox

* Rename _lttoolbox.so

* Fixed import statement

* rename wrappers

* rename: install_language_pack -> install_module

* build: apertium-core

* Implemented changes suggested

Move: _download_apertium_windows() -> install_apertium_base()
TODO: install_wrapper()
remove: Ubuntu.__init__()
add: -y flag during installation on ubuntu
remove args from _rename_wrappers()
rename: get_installer_object() -> get_installer()

* Implemented changes suggested

Install single module at a time

* mock streamparser

import apertium in setup.py,
causes import error, if apertium-streamparser isnt available during installation

* travis: setup.py install before pipenv

* ValueError if unsupported target

* store value of platform.system()

* setup.py: remove mocking

replace: install_requires -> setup_requires allows using streamparser

* perform test before pipenv install

* Rearrange a bit

* close file handles of tempfile

* Simplify output/input file handling

* Fix: tempfile.read() is already byte type

* Snip trailing whitespace

.appveyor.yml

@@ -7,13 +7,13 @@ environment:
       PYTHON_VERSION: 3.5
       PYTHON_ARCH: 32
     - PYTHON: C:\\Python35-x64
-      PYTHON_VERSION: 3.4
+      PYTHON_VERSION: 3.5
       PYTHON_ARCH: 64
     - PYTHON: C:\\Python36
       PYTHON_VERSION: 3.6
       PYTHON_ARCH: 32
     - PYTHON: C:\\Python36-x64
-      PYTHON_VERSION: 3.4
+      PYTHON_VERSION: 3.6
       PYTHON_ARCH: 64
     - PYTHON: C:\\Python37
       PYTHON_VERSION: 3.7

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+omit=apertium/installer.py

.flake8

@@ -2,5 +2,5 @@
 ignore = W504
 max-line-length = 160
 import-order-style = google
-exclude = apertium/swig/, docs/source/conf.py
+exclude = docs/source/conf.py
 application-import-names = apertium

.travis.yml

@@ -12,14 +12,12 @@ python:
     - 'nightly'
 install:
     - pip install pipenv
+    - python3 setup.py install
     - pipenv install --dev --system
 before_script:
-    - wget http://apertium.projectjj.com/apt/install-nightly.sh -O - | sudo bash
-    - sudo apt-get -f --allow-unauthenticated install apertium-all-dev
-    - sudo apt-get -f --allow-unauthenticated install apertium-nno-nob apertium-es-en apertium-eng
-    - python3 setup.py test
     - ./build-swig-wrapper.sh
 script:
+    - python3 setup.py test
     - flake8 --verbose apertium
     - if [[ $TRAVIS_PYTHON_VERSION != 3.4 ]]; then
         mypy apertium --strict --any-exprs-report .mypy_coverage --ignore-missing-imports;

apertium/__init__.py

@@ -5,6 +5,7 @@
 
 from apertium.analysis import analyze, Analyzer  # noqa: F401
 from apertium.generation import generate, Generator  # noqa: F401
+from apertium.installer import install_module  # noqa: F401
 from apertium.mode_search import search_path
 from apertium.translation import translate, Translator  # noqa: F401
 

apertium/installer.py

@@ -0,0 +1,173 @@
+from distutils.dir_util import copy_tree
+import logging
+import os
+import platform
+import shutil
+import subprocess
+import tempfile
+from typing import Optional
+from urllib.request import urlretrieve
+from zipfile import ZipFile
+
+
+class Windows:
+    """Download ApertiumWin64 and Move to %localappdata%"""
+    base_link = 'http://apertium.projectjj.com/{}'
+
+    def __init__(self) -> None:
+        self._install_path = os.getenv('LOCALAPPDATA')
+        self._apertium_path = os.path.join(self._install_path, 'apertium-all-dev')
+        self._download_path = tempfile.mkdtemp()
+        logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
+        self._logger = logging.getLogger()
+        self._logger.setLevel(logging.DEBUG)
+
+    def _download_zip(self, download_files: dict, extract_path: Optional[str]) -> None:
+        for zip_name, zip_link in download_files.items():
+            zip_download_path = os.path.join(self._download_path, zip_name)
+            urlretrieve(Windows.base_link.format(zip_link), filename=zip_download_path)
+            self._logger.info('%s download completed', zip_name)
+
+            # Extract the zip
+            with ZipFile(zip_download_path) as zip_file:
+                zip_file.extractall(path=extract_path)
+            self._logger.info('%s Extraction completed', zip_name)
+            os.remove(zip_download_path)
+            self._logger.info('%s removed', zip_name)
+
+    def _download_package(self, package: str) -> None:
+        """Installs Packages to %localappdata%/Apertium"""
+
+        zip_path = 'win32/nightly/data.php?zip='
+        package_zip = {package: zip_path + package}
+        self._download_zip(package_zip, self._download_path)
+
+        # move the extracted files to desired location
+        lang_data_path = os.path.join(self._download_path, 'usr', 'share', 'apertium')
+
+        self._logger.info('Copying Language Data to Apertium')
+        for directory in os.listdir(lang_data_path):
+            source = os.path.join(lang_data_path, directory)
+            destination = os.path.join(self._apertium_path, 'share', 'apertium', directory)
+            copy_tree(source, destination)
+            self._logger.info('%s -> %s', source, destination)
+
+        shutil.rmtree(os.path.join(self._download_path, 'usr'))
+
+    def _edit_modes(self) -> None:
+        r"""The mode files need to be modified before being used on Windows System
+
+        1. Replace /usr/share with %localappdata%\apertium-all-dev\share
+        2. Replace "/" with "\" to make path compatible with Windows System
+        """
+
+        # List of Mode Files
+        mode_path = os.path.join(self._apertium_path, 'share', 'apertium', 'modes')
+        for f in os.listdir(mode_path):
+            if os.path.isfile(os.path.join(mode_path, f)) and f.endswith('.mode'):
+                self._logger.info('Editing mode %s ', f)
+                with open(os.path.join(mode_path, f)) as infile:
+                    line = infile.read()
+
+                contents = line.split(' ')
+                # Editing mode file to be compatible with windows platform
+                for i, t in enumerate(contents):
+                    if len(t) > 2 and t[0] == "'" and t[1] == '/':
+                        t = t.replace('/', os.sep)
+                        t = t.replace(r'\usr', self._apertium_path)
+                        contents[i] = t
+                line = ' '.join(contents)
+                with open(os.path.join(mode_path, f), 'w') as outfile:
+                    outfile.write(line)
+                    outfile.close()
+
+    def install_apertium_base(self) -> None:
+        """Installs Apertium-all-dev to %localappdata%"""
+
+        apertium_windows = {
+            'apertium-all-dev.zip': '/win64/nightly/apertium-all-dev.zip',
+        }
+
+        self._download_zip(apertium_windows, self._install_path)
+
+    def install_apertium_module(self, language: str) -> None:
+        self._download_package(language)
+        self._edit_modes()
+
+    def install_wrapper(self, swig_wrapper: str) -> None:
+        # TODO: create installer for wrappers on windows
+        pass
+
+
+class Ubuntu:
+    @staticmethod
+    def _install_package_source() -> None:
+        install_script_url = 'http://apertium.projectjj.com/apt/install-nightly.sh'
+        with tempfile.NamedTemporaryFile('w') as install_script:
+            urlretrieve(install_script_url, install_script.name)
+            execute = subprocess.run(['sudo', 'bash', install_script.name])
+            execute.check_returncode()
+
+    @staticmethod
+    def _download_package(package: str) -> None:
+        command = ['sudo', 'apt-get', 'install', '-y', package]
+        execute = subprocess.run(command)
+        execute.check_returncode()
+
+    @staticmethod
+    def _rename_wrappers() -> None:
+        wrapper_name = {
+            'python3-apertium': '_apertium_core',
+            'python3-apertium-lex-tools': '_lextools',
+            'python3-lttoolbox': '_lttoolbox',
+        }
+        dist_package = '/usr/lib/python3/dist-packages'
+        for wrapper in wrapper_name.values():
+            for f in os.listdir(dist_package):
+                if f.startswith(wrapper):
+                    old_name = os.path.join(dist_package, f)
+                    new_name = os.path.join(dist_package, '{}.so'.format(f.split('.')[0]))
+                    if old_name != new_name:
+                        subprocess.run(['sudo', 'mv', old_name, new_name])
+
+    def install_apertium_module(self, language: str) -> None:
+        self._download_package(language)
+
+    def install_apertium_base(self) -> None:
+        self._install_package_source()
+        self._download_package('apertium-all-dev')
+
+    def install_wrapper(self, swig_wrapper: str) -> None:
+        self._download_package(swig_wrapper)
+        self._rename_wrappers()
+
+
+def get_installer():
+    system = platform.system()
+    if system == 'Windows':
+        return Windows()
+    elif system == 'Linux':
+        with open('/etc/os-release') as os_release:
+            distro_name = os_release.readline().split('=')[-1].strip().replace('"', '')
+        if distro_name == 'Ubuntu':
+            return Ubuntu()
+        else:
+            raise ValueError('Installation on {} not supported'.format(distro_name))
+    else:
+        raise ValueError('Installation on {} not supported'.format(system))
+
+
+def install_apertium() -> None:
+    installer = get_installer()
+    installer.install_apertium_base()
+
+
+def install_module(module: str) -> None:
+    apertium_module = 'apertium-{}'.format(module)
+    installer = get_installer()
+    installer.install_apertium_module(apertium_module)
+
+
+def install_wrapper(swig_wrapper: str) -> None:
+    installer = get_installer()
+    installer.install_wrapper(swig_wrapper)

apertium/utils.py

@@ -1,9 +1,14 @@
 import os
+import platform
 import subprocess
+import sys
 import tempfile
 from typing import List
 
 try:
+    if platform.system() == 'Linux':
+        sys.path.append('/usr/lib/python3/dist-packages')
+
     import apertium_core
     import lextools
     import lttoolbox
@@ -46,14 +51,16 @@ def execute_pipeline(inp: str, commands: List[List[str]]) -> str:
         # delete=False and manually delete the file.
         used_wrapper = True
         if wrappers_available:
-            input_file = tempfile.NamedTemporaryFile(delete=False)
+            input_file = tempfile.NamedTemporaryFile(delete=False, mode='w')
             output_file = tempfile.NamedTemporaryFile(delete=False)
+            input_file_name, output_file_name = input_file.name, output_file.name
+
             arg = command[1][1] if len(command) >= 3 else ''
             path = command[-1]
-            input_file_name, output_file_name = input_file.name, output_file.name
-            with open(input_file_name, 'w') as input_file:
-                text = end.decode()
-                input_file.write(text)
+            text = end.decode()
+            input_file.write(text)
+            input_file.close()
+
             if 'lt-proc' == command[0]:
                 lttoolbox.LtLocale.tryToSetLocale()
                 fst = lttoolbox.FST()
@@ -77,16 +84,21 @@ def execute_pipeline(inp: str, commands: List[List[str]]) -> str:
             elif 'apertium-pretransfer' == command[0]:
                 obj = apertium_core.apertium()
                 obj.pretransfer(arg, input_file.name, output_file.name)
+            elif 'apertium-tagger' == command[0]:
+                command += [input_file.name, output_file.name]
+                apertium_core.tagger(len(command), command)
             else:
                 used_wrapper = False
+
             if used_wrapper:
-                with open(output_file_name) as output_file:
-                    end = output_file.read().encode()
+                output_file.seek(0)
+                end = output_file.read()
+            output_file.close()
+
             os.remove(input_file_name)
             os.remove(output_file_name)
         if not wrappers_available or not used_wrapper:
-            if not used_wrapper:
-                apertium.logger.warning('Calling subprocess %s', command[0])
+            apertium.logger.warning('Calling subprocess %s', command[0])
             proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
             end, _ = proc.communicate(end)
     return end.decode()