Introducing logfile checksumming + various fixes

.gitignore

@@ -0,0 +1,3 @@
+*.retry
+hosts
+ansible.cfg

ansible/ansible.cfg.template

@@ -0,0 +1,8 @@
+[defaults]
+inventory = hosts
+stdout_callback = yaml # makes error output more readable
+
+[ssh_connection]
+pipelining = True
+# enable this if ansible hangs on long-running tasks
+#ssh_args = -o ServerAliveInterval=100 -o ControlMaster=auto -o ControlPersist=60m

ansible/hosts.template

@@ -0,0 +1,32 @@
+[ovh]
+ovh-vpsssd1-ubuntu        ansible_host=X.X.X.X
+
+[linode]
+linode-linode1024-ubuntu  ansible_host=X.X.X.X
+
+[digitalocean]
+do-5bucks-ubuntu          ansible_host=X.X.X.X
+
+[scaleway]
+scaleway-vc1s-ubuntu      ansible_host=X.X.X.X
+
+[vultr]
+vultr-20gbssd-ubuntu      ansible_host=X.X.X.X
+vultr-25gbssd-ubuntu      ansible_host=X.X.X.X
+
+
+# vars by provider
+[ovh:vars]
+ansible_user=root
+
+[linode:vars]
+ansible_user=root
+
+[digitalocean:vars]
+ansible_user=root
+
+[scaleway:vars]
+ansible_user=root
+
+[vultr:vars]
+ansible_user=root

ansible/roles/common/tasks/main.yml

@@ -6,3 +6,20 @@
   become: no
   tags:
     - gather
+
+- name: calculate sha512 checksum of results table
+  stat:
+    checksum_algorithm: sha512
+    get_checksum: true
+    get_attributes: false
+    get_md5: false
+    get_mime: false
+    path: ../logs/tables.org
+  register: logfile
+  delegate_to: localhost
+  become: no
+
+- name: save checksum to file
+  local_action:
+    copy content="{{logfile.stat.checksum}}" dest="../logs/tables.org.sha512"
+  become: no

ansible/roles/compiler/tasks/main.yml

@@ -2,39 +2,63 @@
 
 - name: download mariadb
   get_url:
-    url:  https://downloads.mariadb.org/f/mariadb-10.1.22/source/mariadb-10.1.22.tar.gz?serve
-    dest: /tmp/mariadb.tar.gz
+    url: https://downloads.mariadb.com/MariaDB/mariadb-{{mariadb_version}}/source/mariadb-{{mariadb_version}}.tar.gz
+    dest: /tmp/mariadb-{{mariadb_version}}.tar.gz
 
 - name: uncompress mariadb
   unarchive:
-    src: /tmp/mariadb.tar.gz
+    src: /tmp/mariadb-{{mariadb_version}}.tar.gz
     dest: /tmp
     copy: no
 
-- name: set source repo
-  apt_repository:
-    repo: "deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates universe"
-    state: present
-
-- name: install packages need to build mariadb
+- name: install packages needed to build mariadb
   apt:
     name: "{{item}}"
-    state: build-dep
+    state: present
     update_cache: yes
     install_recommends: no
   with_items:
-    - mariadb-server
+    - bison
+    - make
+    - cmake
+    - g++
+    - gcc
+    - git
+    - libaio-dev
+    - libcurl4-gnutls-dev
+    - libevent-dev
+    - libgnutls28-dev
+    - libjemalloc-dev
+    - libncurses-dev
+    - libxml2-dev
+    - zlib1g-dev
 
 - name: configure the mariadb build
-  shell: cmake . -DBUILD_CONFIG=mysql_release
+  shell: cmake . -DBUILD_CONFIG=mysql_release -DWITHOUT_MROONGA_STORAGE_ENGINE=YES
   args:
-    chdir: /tmp/mariadb-10.1.22
+    chdir: /tmp/mariadb-{{mariadb_version}}
+  # some versions of gcc segfault when building mroonga, so omit it
 
 - name: compile mariadb
   shell: (time -p make) > /tmp/compiler.log 2>&1
   args:
     executable: /bin/bash
-    chdir: /tmp/mariadb-10.1.22
+    chdir: /tmp/mariadb-{{mariadb_version}}
+
+- name: calculate sha512 checksum of logfile
+  stat:
+    checksum_algorithm: sha512
+    get_checksum: true
+    get_attributes: false
+    get_md5: false
+    get_mime: false
+    path: /tmp/compiler.log
+  register: logfile
+
+- name: save checksum to file
+  local_action:
+    copy content="{{ logfile.stat.checksum }}" dest="../logs/{{inventory_hostname}}/compiler.log.sha512"
+  become: no
 
 - name: retrieve log
   fetch:
@@ -44,11 +68,11 @@
 
 - name: remove mariadb files
   file:
-    path: /tmp/test
+    path: "{{item}}"
     state: absent
   with_items:
-    - /tmp/mariadb-10.1.22
-    - /tmp/mariadb.tar.gz
+    - /tmp/mariadb-{{mariadb_version}}
+    - /tmp/mariadb-{{mariadb_version}}.tar.gz
 
 - name: remove installed packages
   apt:
@@ -57,83 +81,17 @@
     purge: yes
     state: absent
   with_items:
-    - autotools-dev
-    - binutils
     - bison
-    - build-essential
-    - chrpath
+    - make
     - cmake
-    - cmake-data
-    - cpp
-    - cpp-5
-    - debhelper
-    - dh-apparmor
-    - dh-strip-nondeterminism
-    - dpkg-dev
     - g++
-    - g++-5
     - gcc
-    - gcc-5
-    - gdb
-    - gettext
-    - icu-devtools
-    - intltool-debian
+    - git
     - libaio-dev
-    - libaio1
-    - libarchive-zip-perl
-    - libarchive13
-    - libasan2
-    - libatomic1
-    - libbabeltrace-ctf1
-    - libbabeltrace1
-    - libbison-dev
-    - libboost-dev
-    - libboost1.58-dev
-    - libc-dev-bin
-    - libc6-dev
-    - libcc1-0
-    - libcilkrts5
-    - libcroco3
-    - libcurl3
-    - libdpkg-perl
-    - libfile-stripnondeterminism-perl
-    - libgcc-5-dev
-    - libgomp1
-    - libicu-dev
-    - libisl15
-    - libitm1
+    - libcurl4-gnutls-dev
+    - libevent-dev
+    - libgnutls28-dev
     - libjemalloc-dev
-    - libjemalloc1
-    - libjsoncpp1
-    - libjudy-dev
-    - libjudydebian1
-    - liblsan0
-    - libltdl-dev
-    - libltdl7
-    - libmpc3
-    - libmpx0
-    - libncurses5-dev
-    - libodbc1
-    - libpam0g-dev
-    - libpcre16-3
-    - libpcre3-dev
-    - libpcre32-3
-    - libpcrecpp0v5
-    - libquadmath0
-    - libreadline-gplv2-dev
-    - libstdc++-5-dev
-    - libtimedate-perl
-    - libtinfo-dev
-    - libtsan0
-    - libubsan0
-    - libunistring0
+    - libncurses-dev
     - libxml2-dev
-    - linux-libc-dev
-    - m4
-    - make
-    - odbcinst
-    - odbcinst1debian2
-    - po-debconf
-    - unixodbc
-    - unixodbc-dev
-    - zlib1g-dev 
+    - zlib1g-dev

ansible/roles/dd/tasks/main.yml

@@ -6,26 +6,43 @@
     chdir: /tmp
   with_sequence: count=5
 
-- name: retrieve log
-  fetch:
-    src: /tmp/dd_cpu.log
-    dest: ../logs/{{inventory_hostname}}/dd_cpu.log
-    flat: yes
-
 - name: run dd IO test
   shell: dd if=/dev/zero of=test bs=64k count=16k conv=fdatasync 2>> dd_io.log
   args:
     chdir: /tmp
   with_sequence: count=5
 
-- name: retrieve log
+- name: calculate sha512 checksums of logfiles
+  stat:
+    checksum_algorithm: sha512
+    get_checksum: true
+    get_attributes: false
+    get_md5: false
+    get_mime: false
+    path: /tmp/{{item}}.log
+  register: logfile
+  with_items:
+    - dd_io
+    - dd_cpu
+
+- name: save checksums to files
+  local_action:
+    copy content="{{item.stat.checksum}}" dest="../logs/{{inventory_hostname}}/{{item.item}}.log.sha512"
+  become: no
+  with_items:
+    - "{{ logfile.results }}"
+
+- name: retrieve logs
   fetch:
-    src: /tmp/dd_io.log
-    dest: ../logs/{{inventory_hostname}}/dd_io.log
+    src: /tmp/{{item}}.log
+    dest: ../logs/{{inventory_hostname}}/{{item}}.log
     flat: yes
+  with_items:
+    - dd_io
+    - dd_cpu
 
 - name: remove auxiliary file
-  file: 
+  file:
     path: /tmp/test
     state: absent
 

ansible/roles/downloads/tasks/main.yml

@@ -7,7 +7,22 @@
   shell: cat downloads_1.log downloads_2.log downloads_3.log > downloads.log
   args:
     chdir: /tmp
-
+
+- name: calculate sha512 checksum of logfile
+  stat:
+    checksum_algorithm: sha512
+    get_checksum: true
+    get_attributes: false
+    get_md5: false
+    get_mime: false
+    path: /tmp/downloads.log
+  register: logfile
+
+- name: save checksum to file
+  local_action:
+    copy content="{{ logfile.stat.checksum }}" dest="../logs/{{inventory_hostname}}/downloads.log.sha512"
+  become: no
+
 - name: retrieve log
   fetch:
     src: /tmp/downloads.log