Switch to Python 3 **only** (>= 3.6)

Also, update Docker images

.github/CONTRIBUTING.md

@@ -80,9 +80,7 @@ dependency makes it harder for users to install IVRE.
 
 We try to comply with the some guidelines for new code:
 
--   The code **must** be compatible with Python 2.7 and 3.4 to
-    3.8. The [future module](http://python-future.org/) can help (IVRE
-    already depends on it, so feel free to use it in your code).
+-   The code **must** be compatible with Python 3.6 to 3.9.
 
 -   The code should be PEP-8 compliant; you can check your code with
     [pep8](https://pypi.python.org/pypi/pep8).
@@ -101,7 +99,6 @@ We try to comply with the some guidelines for new code:
 -   Avoid creating `list` objects when generators can be used,
     particularly if they can be huge:
 
-    -   Make sure to import `range` from `future.utils` (for Python 2).
     -   Use `for line in fdesc` instead of `for line in fdesc.readlines()`.
     -   More generally, prefer generators over lists.
 

.travis.yml

@@ -38,42 +38,8 @@ language: python
 cache: pip
 
 python:
-  - 2.7
-  - 3.5
-  - 3.8
-
-matrix:
-  # ivre ipdata --download segfaults with Bionic, Python 3.5 and PostgreSQL...
-  exclude:
-    - python: 3.5
-      env: DB=postgres POSTGRES_VERSION=9.5.10
-    - python: 3.5
-      env: DB=postgres POSTGRES_VERSION=10.1
-  include:
-    - python: 3.4
-      dist: xenial
-      env: DB=maxmind
-    - python: 3.4
-      dist: xenial
-      env: DB=tinydb
-    - python: 3.4
-      dist: xenial
-      env: DB=elastic ELASTIC_VERSION=7.3.1
-    - python: 3.4
-      dist: xenial
-      env: DB=postgres POSTGRES_VERSION=9.5.10
-    - python: 3.4
-      dist: xenial
-      env: DB=postgres POSTGRES_VERSION=10.1
-    - python: 3.4
-      dist: xenial
-      env: DB=mongo MONGODB_VERSION=3.2.21
-    - python: 3.4
-      dist: xenial
-      env: DB=mongo MONGODB_VERSION=4.1.3
-    - python: 3.4
-      dist: xenial
-      env: DB=sqlite
+  - 3.6
+  - 3.9
 
 # PhantomJS is supposed to be installed
 # (https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-PhantomJS),
@@ -130,7 +96,7 @@ before_script:
   - export ZEEK_SAMPLES=`pwd`/usr/local/zeek/testing
   - ivre --version; echo; zeek --version; echo; nmap --version
 
-script: (test "$DB" != "maxmind" || (flake8 --ignore=F401,E402 ./doc/conf.py && flake8 --ignore=W504 setup.py ./bin/ivre && flake8 --ignore=E402,W504 ./tests/tests.py && flake8 --ignore=W504 ./ivre.bak/ && echo "flake8 OK (except W504)")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.8 || (codespell  --ignore-words=.travis/codespell_ignore `git ls-files | grep -vE '^web/static/(doc|an|bs|d3|jq|lk)/|^data/|\.(png|gif|svg)$'` && echo "codespell OK")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.8 || (pylint -e all -d abstract-method,arguments-differ,attribute-defined-outside-init,bad-continuation,broad-except,cyclic-import,deprecated-module,duplicate-code,exec-used,fixme,function-redefined,global-statement,global-variable-undefined,import-error,import-outside-toplevel,invalid-name,locally-disabled,missing-docstring,no-absolute-import,no-member,non-parent-init-called,no-self-use,old-division,protected-access,raise-missing-from,reload-builtin,round-builtin,singleton-comparison,subprocess-popen-preexec-fn,super-init-not-called,super-with-arguments,suppressed-message,too-few-public-methods,too-many-ancestors,too-many-arguments,too-many-boolean-expressions,too-many-branches,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-nested-blocks,too-many-public-methods,too-many-return-statements,too-many-statements,unsubscriptable-object,unsupported-membership-test,unused-argument,useless-object-inheritance,wrong-import-order,wrong-import-position ivre && echo "pylint OK")) && cd tests/ && coverage erase && coverage run --parallel-mode tests.py --coverage && coverage combine && coverage report -i
+script: (test "$DB" != "maxmind" || (flake8 --ignore=F401,E402 ./doc/conf.py && flake8 --ignore=W504 setup.py ./bin/ivre && flake8 --ignore=E402,W504 ./tests/tests.py && flake8 --ignore=W504 ./ivre.bak/ && echo "flake8 OK (except W504)")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.9 || (codespell  --ignore-words=.travis/codespell_ignore `git ls-files | grep -vE '^web/static/(doc|an|bs|d3|jq|lk)/|^data/|\.(png|gif|svg)$'` && echo "codespell OK")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.9 || (pylint -e all -d abstract-method,arguments-differ,attribute-defined-outside-init,bad-continuation,broad-except,cyclic-import,deprecated-module,duplicate-code,exec-used,fixme,function-redefined,global-statement,global-variable-undefined,import-error,import-outside-toplevel,invalid-name,locally-disabled,missing-docstring,no-absolute-import,no-member,non-parent-init-called,no-self-use,old-division,protected-access,raise-missing-from,round-builtin,singleton-comparison,subprocess-popen-preexec-fn,super-init-not-called,super-with-arguments,suppressed-message,too-few-public-methods,too-many-ancestors,too-many-arguments,too-many-boolean-expressions,too-many-branches,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-nested-blocks,too-many-public-methods,too-many-return-statements,too-many-statements,unsubscriptable-object,unsupported-membership-test,unused-argument ivre && echo "pylint OK")) && cd tests/ && coverage erase && coverage run --parallel-mode tests.py --coverage && coverage combine && coverage report -i
 
 after_success:
   - codecov

.travis/install_maxmind.sh

@@ -17,8 +17,8 @@
 # along with IVRE. If not, see <http://www.gnu.org/licenses/>.
 
 PIP_INSTALL_OPTIONS=""
-# We only run codespell and pylint once, with Python 3.8
-if [ "$TRAVIS_PYTHON_VERSION" = 3.8 ]; then
+# We only run codespell and pylint once, with Python 3.9
+if [ "$TRAVIS_PYTHON_VERSION" = 3.9 ]; then
     pip install -U $PIP_INSTALL_OPTIONS -r requirements-linting.txt
 else
     pip install -U $PIP_INSTALL_OPTIONS flake8

doc/dev/linting.rst

@@ -31,5 +31,5 @@ Travis CI
 
 Code linting and spell checking is performed in Travis CI (see
 :ref:`dev/tests:Travis CI`), togethter with the Maxmind tests. Pylint
-and Codespell only run with Python 3.8, while Flake8 runs with all
+and Codespell only run with Python 3.9, while Flake8 runs with all
 Python versions.

doc/dev/tests.rst

@@ -46,7 +46,7 @@ place your samples there (the PCAP files must have the extension
 and the Nmap JSON results must have the extension ``.json``).
 
 Then, run ``python ./tests.py`` (optionally replace ``python`` by the
-alternative interpreter you want to use, e.g., ``python2.7``; note
+alternative interpreter you want to use, e.g., ``python3.9``; note
 that ``coverage.py`` must be installed for this interpreter).  The
 first run will create a ``samples/results`` file with the expected
 values for some results. The next runs will use those values to check
@@ -87,6 +87,6 @@ Travis CI
 
 Tests are run with several MongoDB and PostgreSQL versions, as well as
 TinyDB, SQLite and Elasticsearch for each pull requests. The tests run
-with Python 2.7 and 3.4 to 3.8.
+with Python 3.6 to 3.9.
 
 The configuration is in the `.travis.yml` YAML file.

doc/install/docker.rst

@@ -102,7 +102,7 @@ run:
 ::
 
    $ git archive --format=tar --prefix=ivre/ HEAD -o docker/base-local/ivre.tar
-   $ tmp=`mktemp | sed 's#^/##'`; python2 setup.py --version | tr -d '\n' > "/$tmp"
+   $ tmp=`mktemp | sed 's#^/##'`; python setup.py --version | tr -d '\n' > "/$tmp"
    $ tar rf docker/base-local/ivre.tar --transform="s#$tmp#ivre/ivre/VERSION#" /$tmp
    $ rm "/$tmp"
    $ docker pull debian:stable

doc/install/fast-install-and-first-run.rst

@@ -14,7 +14,7 @@ referring to their documentation (see `MongoDB on Debian
 or `MongoDB on Ubuntu
 <http://docs.mongodb.org/manual/tutorial/install-mongodb-on-ubuntu/>`__
 and `pymongo <https://pypi.python.org/pypi/pymongo/>`__), instead of
-``apt-get``-ting them.
+getting them via ``apt``.
 
 You might also want to adapt it to your needs, architecture, etc.
 
@@ -26,12 +26,12 @@ Install
 
 ::
 
-   $ sudo apt-get -y install mongodb python-pymongo python-crypto \
-   >   python-future python-bottle apache2 libapache2-mod-wsgi dokuwiki
+   $ sudo apt -y --no-install-recommends install mongodb python3-pymongo \
+   >   python3-crypto python3-bottle apache2 libapache2-mod-wsgi-py3 dokuwiki
    $ git clone https://github.com/cea-sec/ivre
    $ cd ivre
-   $ python setup.py build
-   $ sudo python setup.py install
+   $ python3 setup.py build
+   $ sudo python3 setup.py install
 
 Setup
 -----

doc/install/installation.rst

@@ -40,12 +40,11 @@ your needs:
 Python
 ~~~~~~
 
-To install IVRE, you'll need `Python <http://www.python.org/>`__ 2.7
-or 3 (version 3.4 minimum), with the following modules:
+To install IVRE, you'll need `Python <http://www.python.org/>`__ 3.6
+minimum, with the following modules:
 
 - `bottle <https://bottlepy.org/>`_.
 - `cryptography <https://cryptography.io/en/latest/>`_.
-- `future <https://python-future.org/>`_.
 - `pymongo <http://api.mongodb.org/python/>`_ version 2.7.2 minimum.
 - `tinydb <https://tinydb.readthedocs.io/>`_, to use the
   **experimental** TinyDB backend (this does not require a database
@@ -132,13 +131,11 @@ The installation of IVRE itself can be done:
 
    - ``ivre``: the main package, which depends on ``python-ivre``.
 
-   - ``python-ivre`` and ``python2-ivre``: the Python 3 and Python 2
-     libraries. You don't need ``python2-ivre`` unless you have Python
-     2 only code relying on IVRE.
+   - ``python-ivre`` the Python library.
 
-   - ``ivre-web``: the Web application
+   - ``ivre-web``: the Web application.
 
-   - ``ivre-docs``: the documentation
+   - ``ivre-docs``: the documentation.
 
   These packages are based on the latest stable version; they all have
   a ``-git`` version, based on the current development code from the

docker/agent/Dockerfile

@@ -1,5 +1,5 @@
 # This file is part of IVRE.
-# Copyright 2011 - 2018 Pierre LALET <pierre.lalet@cea.fr>
+# Copyright 2011 - 2020 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -15,7 +15,7 @@
 # along with IVRE. If not, see <http://www.gnu.org/licenses/>.
 
 FROM ivre/client
-LABEL maintainer="Pierre LALET <pierre.lalet@cea.fr>"
+LABEL maintainer="Pierre LALET <pierre@droids-corp.org>"
 
 RUN ivre runscans --output Agent > /usr/bin/agent && \
     chmod +x /usr/bin/agent

docker/base-local/Dockerfile

@@ -1,5 +1,5 @@
 # This file is part of IVRE.
-# Copyright 2011 - 2019 Pierre LALET <pierre.lalet@cea.fr>
+# Copyright 2011 - 2020 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -15,20 +15,21 @@
 # along with IVRE. If not, see <http://www.gnu.org/licenses/>.
 
 FROM debian:stable
-LABEL maintainer="Pierre LALET <pierre.lalet@cea.fr>"
+LABEL maintainer="Pierre LALET <pierre@droids-corp.org>"
 
 ENV DEBIAN_FRONTEND noninteractive
 
 # Dependencies
 RUN echo "deb http://deb.debian.org/debian stable-backports main" >> /etc/apt/sources.list
-RUN apt-get -q update
-RUN apt-get -qy install python-pymongo python-crypto python-setuptools \
-    python-future python-bottle ca-certificates
+RUN apt-get -q update && \
+    apt-get -qy --no-install-recommends install python3-pymongo python3-crypto \
+        python3-setuptools python3-bottle ca-certificates && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # IVRE
 ADD ivre.tar ./
 RUN cd ivre/ && \
-    python setup.py build && python setup.py install && \
+    python3 setup.py build && python3 setup.py install && \
     cd ../ && rm -rf ivre/
 
 RUN echo 'DB = "mongodb://ivredb/"' > /etc/ivre.conf

docker/base-pip/Dockerfile

@@ -1,5 +1,5 @@
 # This file is part of IVRE.
-# Copyright 2011 - 2019 Pierre LALET <pierre.lalet@cea.fr>
+# Copyright 2011 - 2020 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -15,14 +15,15 @@
 # along with IVRE. If not, see <http://www.gnu.org/licenses/>.
 
 FROM debian:stable
-LABEL maintainer="Pierre LALET <pierre.lalet@cea.fr>"
+LABEL maintainer="Pierre LALET <pierre@droids-corp.org>"
 
 ENV DEBIAN_FRONTEND noninteractive
 
 # Install pip, get IVRE, uninstall pip
-RUN apt-get -q update
-RUN apt-get -qy install python python-dev python-pip && \
-    pip install ivre && \
-    apt-get -qy autoremove python-dev python-pip
+RUN apt-get -q update && \
+    apt-get -qy --no-install-recommends install python3 python3-dev python3-pip python3-setuptools && \
+        pip3 install ivre && \
+    apt-get -qy autoremove python3-dev python3-pip python3-setuptools && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN echo 'DB = "mongodb://ivredb/"' > /etc/ivre.conf

docker/base/Dockerfile

@@ -1,5 +1,5 @@
 # This file is part of IVRE.
-# Copyright 2011 - 2019 Pierre LALET <pierre.lalet@cea.fr>
+# Copyright 2011 - 2020 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -15,19 +15,23 @@
 # along with IVRE. If not, see <http://www.gnu.org/licenses/>.
 
 FROM debian:stable
-LABEL maintainer="Pierre LALET <pierre.lalet@cea.fr>"
+LABEL maintainer="Pierre LALET <pierre@droids-corp.org>"
 
 ENV DEBIAN_FRONTEND noninteractive
 
 # Dependencies
 RUN echo "deb http://deb.debian.org/debian stable-backports main" >> /etc/apt/sources.list
-RUN apt-get -q update
-RUN apt-get -qy install python-pymongo python-crypto python-setuptools \
-    python-future python-bottle ca-certificates
+RUN apt-get -q update && \
+    apt-get -qy --no-install-recommends install python3-pymongo python3-crypto \
+        python3-setuptools python3-bottle ca-certificates && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # IVRE
-RUN apt-get -qy install git && git clone https://github.com/cea-sec/ivre && \
-    cd ivre/ && python setup.py build && python setup.py install && \
-    apt-get -qy --purge autoremove git && cd ../ && rm -rf ivre/
+RUN apt-get -q update && \
+    apt-get -qy --no-install-recommends install git && \
+    git clone https://github.com/cea-sec/ivre && \
+    cd ivre/ && python3 setup.py build && python3 setup.py install && \
+    apt-get -qy --purge autoremove git && cd ../ && rm -rf ivre/ && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN echo 'DB = "mongodb://ivredb/"' > /etc/ivre.conf

docker/client/Dockerfile

@@ -20,19 +20,21 @@ LABEL maintainer="Pierre LALET <pierre@droids-corp.org>"
 # Tools
 ## non-free: s3270
 RUN sed -i 's/ main/ main non-free/' /etc/apt/sources.list
-RUN apt-get -q update
 ## openssl: IVRE depends on openssl exec + libssl needed for Nmap
 ## libfreetype6 libfontconfig1 fonts-dejavu: screenshots w/ phantomjs
-RUN apt-get -qy install rsync screen ipython openssl tesseract-ocr \
-    libfreetype6 libfontconfig1 fonts-dejavu imagemagick ffmpeg s3270 \
-    patch bash-completion bzip2 bro python-pil nfdump
+RUN apt-get -q update && \
+    apt-get -qy --no-install-recommends install rsync screen ipython3 openssl \
+        tesseract-ocr libfreetype6 libfontconfig1 fonts-dejavu imagemagick \
+        ffmpeg s3270 patch bash-completion bzip2 python3-pil nfdump && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Install Nmap. Use included libpcap as a workaround for Nmap issue
 # #34 (https://github.com/nmap/nmap/issues/34) since we do not know
 # which kernel version will be used
 # ADD https://github.com/nmap/nmap/tarball/master ./nmap.tar.gz
-ADD https://nmap.org/dist/nmap-7.80.tgz ./nmap.tar.gz
-RUN apt-get -qy install build-essential libssl-dev flex bison && \
+ADD https://nmap.org/dist/nmap-7.91.tgz ./nmap.tar.gz
+RUN apt-get -q update && \
+    apt-get -qy --no-install-recommends install build-essential libssl-dev flex bison && \
     tar zxf nmap.tar.gz && \
     mv nmap-* nmap && \
     cd nmap && \
@@ -41,7 +43,17 @@ RUN apt-get -qy install build-essential libssl-dev flex bison && \
                 --with-libpcap=included && \
     make && make install && \
     cd ../ && rm -rf nmap nmap.tar.gz && \
-    apt-get -qy --purge autoremove build-essential libssl-dev flex bison
+    apt-get -qy --purge autoremove build-essential libssl-dev flex bison && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Install Zeek
+RUN apt-get -q update && \
+    apt-get -qy --no-install-recommends install gnupg wget && \
+    echo 'deb http://download.opensuse.org/repositories/security:/zeek/Debian_10/ /' > /etc/apt/sources.list.d/zeek.list && \
+    wget -qO - https://download.opensuse.org/repositories/security:zeek/Raspbian_10/Release.key | apt-key add - && \
+    apt-get -q update && apt-get -qy --no-install-recommends install zeek && \
+    apt-get -qy --purge autoremove gnupg wget && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # "Install" phantomjs for our http-screenshot NSE script replacement
 ADD https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 ./phantomjs-1.9.8-linux-x86_64.tar.bz2

docker/db/Dockerfile

@@ -1,5 +1,5 @@
 # This file is part of IVRE.
-# Copyright 2011 - 2019 Pierre LALET <pierre.lalet@cea.fr>
+# Copyright 2011 - 2020 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -15,17 +15,19 @@
 # along with IVRE. If not, see <http://www.gnu.org/licenses/>.
 
 FROM debian:buster
-LABEL maintainer="Pierre LALET <pierre.lalet@cea.fr>"
+LABEL maintainer="Pierre LALET <pierre@droids-corp.org>"
 
 ENV DEBIAN_FRONTEND noninteractive
 
 # MongoDB
 # https://docs.mongodb.com/manual/tutorial/install-mongodb-on-debian/
-RUN apt-get -q update && apt-get -qy install --no-install-recommends gnupg && \
-    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 4B7C549A058F8B6B && \
-    echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.2 main" | tee /etc/apt/sources.list.d/mongodb-org-4.2.list && \
-    apt-get -q update && apt-get -qy install --no-install-recommends mongodb-org && \
-    apt-get -qy --purge autoremove gnupg && \
+RUN apt-get -q update && \
+    apt-get -qy --no-install-recommends install gnupg wget ca-certificates && \
+    wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - && \
+    echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/4.4 main" > /etc/apt/sources.list.d/mongodb-org-4.4.list && \
+    ln -s /bin/true /usr/bin/systemctl && \
+    apt-get -q update && apt-get -qy --no-install-recommends install mongodb-org && \
+    apt-get -qy --purge autoremove gnupg wget ca-certificates && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Data & log directories

docker/docker-compose.yml

@@ -1,5 +1,5 @@
 # This file is part of IVRE.
-# Copyright 2011 - 2019 Pierre LALET <pierre.lalet@cea.fr>
+# Copyright 2011 - 2020 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -38,3 +38,5 @@ services:
       - ./ivre-share:/ivre-share
     depends_on:
       - ivredb
+    stdin_open: true
+    tty: true