PR pour tester le nouveau script travis + selection de l'OS lors de l'installation

.travis.yml

@@ -7,19 +7,6 @@ python:
   - 3.5
 
 
-addons:
-  firefox: "latest"
-  apt:
-    packages:
-      - mysql-server-5.6
-      - mysql-client-core-5.6
-      - mysql-client-5.6
-      - libmysqlclient-dev
-      - language-pack-fr
-      - unzip
-      - oracle-java9-set-default
-
-
 git:
   depth: 1
 
@@ -31,12 +18,20 @@ matrix:
 env:
   global:
     - secure: "azmDZZQZzf88zpbkYpLpxI66vpEVyv+kniW0QdWAt4qlys8P5OcO3VJBR5YT85vlvnjN9b6raWQAL1ymee0WmVdTmzXed8XjZv7t9QXVw7pfezxMKlEftVp/4Cu4wtvbew0ViZXNWV2hNXHzEqlhgnoIOq94i0UzZ7grMrI0xm0="
-  matrix:
-    - ZDS_TEST_JOB="zds.tutorialv2"
-    - ZDS_TEST_JOB="zds.member zds.utils zds.forum"
-    - ZDS_TEST_JOB="front zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2"
-    - ZDS_TEST_JOB="selenium"
-    - ZDS_TEST_JOB="doc"
+
+jobs:
+  include:
+    # - name: tuto
+    #   env: ZDS_TEST_JOB="zds.tutorialv2"
+    # - name: "test1 : member utils forum"
+    #   env: ZDS_TEST_JOB="zds.member zds.utils zds.forum"
+    # - name: "test2 : front mp gallery pages featured notification searchv2"
+    #   env: ZDS_TEST_JOB="front zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2"
+    # - name: selenium
+    #   env: ZDS_TEST_JOB="selenium"
+    - name: doc
+      env: ZDS_TEST_JOB="doc"
+
 
 notifications:
   webhooks:
@@ -62,120 +57,21 @@ cache:
 
 
 before_install:
-  - . ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced
+  - HACK_SphinxBuild=$VIRTUAL_ENV
 
   - source ./scripts/define_variable.sh
 
-  - travis_retry pip install -q coveralls
-
-  - |
-    # install elasticsearch
-    if [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then
-      # see https://docs.travis-ci.com/user/database-setup/#Installing-specific-versions-of-ElasticSearch
-      curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ZDS_ELASTIC_VERSION.deb \
-      && sudo dpkg -i --force-confnew elasticsearch-$ZDS_ELASTIC_VERSION.deb \
-      && sudo service elasticsearch start
-    fi
-
-  - |
-    # setup mysql
-    if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then
-      # MySQL config
-      ./scripts/ci_mysql_setup.sh
-    fi
-
 install:
-  - |
-    # install webdriver for selenium.
-    if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then
-      wget https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz \
-      && mkdir geckodriver \
-      && tar -xzf geckodriver-v0.23.0-linux64.tar.gz -C geckodriver \
-      && export PATH=$PATH:$PWD/geckodriver \
-      && export DISPLAY=:99.0
-    fi
-
-  - |
-    if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then
-      # install latex
-      git clone $ZDS_LATEX_REPO
-      TEMPLATEDIR=$HOME/.texlive/texmf-local/tex/latex/
-      ./latex-template/scripts/install_font.sh \
-      && ./latex-template/scripts/install_texlive.sh \
-      && export PATH=$HOME/.texlive/bin/x86_64-linux:$PATH \
-      && rm -rf $TEMPLATEDIR/latex-template \
-      && mkdir -p $TEMPLATEDIR  && cp -r ./latex-template $TEMPLATEDIR && texhash
-    fi
-
-  - |
-    # install and run zmarkdown
-    nvm install $ZDS_NODE_VERSION \
-    && nvm use $ZDS_NODE_VERSION \
-    && npm i -g pm2 zmarkdown \
-    && cd "$(nvm which current | sed s/bin/lib/)_modules/zmarkdown/" \
-    && npm run server \
-    && cd $TRAVIS_BUILD_DIR
-
-  - |
-    # install frontend dependencies
-    npm i -g yarn \
-    && yarn --silent
-
-  - |
-    # install backend dependencies
-    if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]] || [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then
-      pip install -q -r requirements-dev.txt -r requirements-prod.txt
-    fi
+  - sudo apt-get update -qq
 
+  - ./scripts/install_zds.sh --answer-yes --travis-output --detect-os-version +packages +virtualenv +back
 
 script:
-  - |
-    # lint backend
-    if [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then
-      ./scripts/no_import_zds_settings.sh \
-      && flake8 \
-      && flake8 --config=zds/settings/.flake8 zds/settings
-    fi
-
-  - |
-    # test and build frontend
-    if [[ "$ZDS_TEST_JOB" == *"front"* ]]; then
-      yarn run lint && yarn run build # we need to upload the assets later
-    fi
-  - |
-    # test backend
-    if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then
-      python manage.py makemigrations --dry-run --check \
-      && coverage run --source='.' manage.py \
-        test -v=2\
-        --keepdb \
-        --settings zds.settings.ci_test \
-        --exclude-tag=front \
-        ${ZDS_TEST_JOB/front/}
-    fi
+  - source ./$ZDS_VENV/bin/activate
 
-  - |
-    # show zmarkdown logs
-    if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then
-      pm2 logs --nostream --raw --lines 1000
-    fi
+  - source $HACK_SphinxBuild/bin/activate
 
-  - |
-    # selenium tests
-    if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then
-      yarn run build
-      xvfb-run --server-args="-screen 0 1280x720x8" python manage.py \
-        test -v=2\
-        --settings zds.settings.ci_test \
-        --tag=front \
-        --keepdb
-    fi
-
-  - |
-    # build documentation
-    if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then
-      make --directory doc html
-    fi
+  - ./scripts/travis_script.sh "build_documentation"
 
 after_success:
   - du -sh $HOME/.texlive 2>/dev/null | true

Makefile

@@ -21,7 +21,10 @@ clean: clean-back clean-front ## Clean everything (`clean-back` & `clean-front`)
 ## ~ Backend
 
 install-back: ## Install the Python packages for the backend
-	pip3 install --upgrade -r requirements-dev.txt
+	pip install --upgrade -r requirements-dev.txt
+
+install-back-with-prod:
+	pip install --upgrade -r requirements-dev.txt -r requirements-prod.txt
 
 run-back: zmd-check ## Run the backend server
 	python manage.py runserver

requirements.txt

@@ -3,6 +3,7 @@ social-auth-app-django==3.1.0
 elasticsearch==5.5.2
 elasticsearch-dsl==5.4.0
 sqlparse==0.2.4
+setuptools==40.6.3
 
 # Explicit dependencies (references in code)
 Django==2.1.5
@@ -11,6 +12,7 @@ django-model-utils==3.1.2
 django-munin==0.2.1
 python-memcached==1.59
 lxml==4.2.5
+libxml2-python3==2.9.5
 factory-boy==2.8.1
 pygeoip==0.3.2
 pillow==5.0.0

scripts/ci_mysql_setup.sh

@@ -1,15 +1,38 @@
 #!/bin/bash
 
-set -ex
-
-sudo sed -i'' 's/\[client\]/\[client\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf
-sudo sed -i'' 's/\[mysql\]/\[mysql\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf
-sudo sed -i'' 's/\[mysqld\]/\[mysqld\]\ninnodb_file_per_table=on\ninnodb_file_format=barracuda\ninnodb_large_prefix=on\ncharacter-set-client-handshake=false\ncharacter-set-server=utf8mb4\ncollation-server=utf8mb4_unicode_ci/' /etc/mysql/my.cnf
-sudo /etc/init.d/mysql restart
-# Travis should fail as soon as possible
-mysql -u root -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES';"
-# Avoid "mysql has gone away" errors
-mysql -u root -e "SET GLOBAL wait_timeout = 36000;"
-mysql -u root -e "SET GLOBAL max_allowed_packet = 134209536;"
-# Create database with the correct charset and collation
-mysql -u root -e "CREATE DATABASE zds_test CHARACTER SET = utf8mb4 COLLATE utf8mb4_unicode_ci;"
+function print_info {
+    echo -n -e "\033[0;36m";
+    echo "$1";
+    echo -n -e "\033[00m";
+}
+
+if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then
+    #display print_info
+    forwho=""
+    if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then
+        forwho="'zds.*' tasks (-> needed for tests)"
+
+        if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then
+            forwho="$forwho and selenium"
+        fi
+    else
+        forwho="selenium"
+    fi
+    print_info "* Install mysql for $forwho."
+
+    #install
+
+    set -ex
+
+    sudo sed -i'' 's/\[client\]/\[client\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf
+    sudo sed -i'' 's/\[mysql\]/\[mysql\]\ndefault-character-set=utf8mb4/' /etc/mysql/my.cnf
+    sudo sed -i'' 's/\[mysqld\]/\[mysqld\]\ninnodb_file_per_table=on\ninnodb_file_format=barracuda\ninnodb_large_prefix=on\ncharacter-set-client-handshake=false\ncharacter-set-server=utf8mb4\ncollation-server=utf8mb4_unicode_ci/' /etc/mysql/my.cnf
+    sudo /etc/init.d/mysql restart
+    # Travis should fail as soon as possible
+    mysql -u root -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES';"
+    # Avoid "mysql has gone away" errors
+    mysql -u root -e "SET GLOBAL wait_timeout = 36000;"
+    mysql -u root -e "SET GLOBAL max_allowed_packet = 134209536;"
+    # Create database with the correct charset and collation
+    mysql -u root -e "CREATE DATABASE zds_test CHARACTER SET = utf8mb4 COLLATE utf8mb4_unicode_ci;"
+fi

scripts/define_variable.sh

@@ -19,3 +19,10 @@ fi
 if [[ $ZDS_LATEX_REPO == "" ]]; then
     ZDS_LATEX_REPO="https://github.com/zestedesavoir/latex-template.git"
 fi
+
+if [[ $ZDS_JDK_HASH == "" ]]; then
+    ZDS_JDK_HASH="f51449fcd52f4d52b93a989c5c56ed3c"
+    # shellcheck disable=SC2034
+    ZDS_JDK_VERSION="11.0.2"
+    ZDS_JDK_REV="+9"
+fi

scripts/dependencies/arch.txt

@@ -0,0 +1,15 @@
+#title=Arch
+#desc=Utilisation de pacman
+#installcmd=pacman -Syu
+git
+wget
+curl
+python
+python3-pip
+libxslt
+zlib
+libffi
+libjpeg-turbo
+freetype2
+base-devel
+unzip

scripts/dependencies/debian.txt

@@ -0,0 +1,20 @@
+#title=Debian
+#desc=Utilisation de apt-get / Testé sur : Debian Jessie
+#installcmd=apt-get -y install
+git
+wget
+curl
+python3-dev
+python3-pip
+python3-venv
+libxslt-dev
+libz-dev
+libjpeg62-turbo
+libjpeg62-turbo-dev
+libfreetype6
+libfreetype6-dev
+libffi-dev
+build-essential
+imagemagick
+librsvg2-bin
+xzdec

scripts/dependencies/fedora.txt

@@ -0,0 +1,18 @@
+#title=Fedora
+#desc=Utilisation de dnf
+#installcmd=dnf -y install
+git
+wget
+curl
+python3-devel
+libxslt-devel
+zlib-devel
+libjpeg-turbo-devel
+libjpeg-turbo-devel
+freetype
+freetype-devel
+libffi-devel
+python3-pip
+python-virtualenv
+gcc
+redhat-rpm-config

scripts/dependencies/ubuntu.txt

@@ -0,0 +1,20 @@
+#title=Ubuntu
+#desc=Utilisation de apt-get / Testé sur : Ubuntu 14.04
+#installcmd=apt-get -y install
+git
+wget
+curl
+python3-dev
+python3-pip
+python3-venv
+libxslt1-dev
+zlib1g-dev
+libjpeg8
+libjpeg8-dev
+libfreetype6
+libfreetype6-dev
+libffi-dev
+build-essential
+imagemagick
+librsvg2-bin
+xzdec