Merge branch 'release/0.4.0'

.github/workflows/python-publish.yml

@@ -9,6 +9,8 @@ on:
   push:
     tags:
       - '*'
+    branches:
+      - develop
 
 jobs:
   deploy:
@@ -20,7 +22,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.x'
+        python-version: '3.8'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

docs/source/images/modelling.drawio

@@ -1 +1 @@
-<mxfile host="Electron" modified="2020-10-25T19:57:03.988Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/13.7.9 Chrome/85.0.4183.121 Electron/10.1.3 Safari/537.36" etag="58WXH3a5L9Q1821uxp0o" version="13.7.9" type="device" pages="5"><diagram id="xTy2y7PyVUbXui-J8zsi" name="Modelling">7Vhdb9owFP01eeyUD0jhkfLRaoWtGpW67mVyY+N4dXLBMU3or59DbJKQDlGplFFV4sH3+OY6Pvcck8Ty+lF2KdA8nAAm3HJtnFnewHJdx3FdK//ZeFUgHdcrACoY1kklMGXPRIO2RpcMk6SWKAG4ZPM6GEAck0DWMCQEpPW0GfD6qnNESQOYBog30TuGZWh2cV7iV4TR0Kzs+N1iJkImWe8kCRGGtAJ5Q8vrCwBZjKKsT3hOnuEFdb5fLCifXJMxuuGjLL12JmdFsdFrLtlsQZBYvm1pvddErgxfBCv6dAhChkAhRnxYohcCljEmeVVHRWXOGGCuwT9EypXWAlpKUFAoI65nZxDLEYoYzyXVEwxxDeornJaK99yz5iaBpQjIjryWlh4SlOyq5xd5OQsV/WhGLwlERIqVShCEI8me6iJDWqt0k1f2Qw10S17RHn3XT4gv9UoDJFGjZ/WOpCGTZDpHaz5S5eqDs/9EhCTZTr7MrDGTPk3cjo7T0pubnLDiy5Z9IIrNOfXxLdA+SQu0GxbocQqCyTD6QD7w/KP7wGnQ+Q4+ULSK1U8V2Ca4z4MvbRMOsurkYKWjY/jH39M/nf/KP37DP/oZ72S94217xz26d9xP7+z2zvlJeue84Z1vJFXAiT+EbRuoZR/bQJ0G0TeCYBZIBnFywky3j/k3P38cTx6nC4/eoh/0hsPvTverfkesMm25PlerXjyoAZVrMgogJ6lGvL9Ygpk4S9b09VSCIi4rJ02VW4FYzGJqqqm7LQrWF1FwZeGtRiuWZb2biRTwSPrAQSgkhjg/SWeM8y0IcUZjFXIyyyvkHWMB4j0NRwzj9SH8knhKeeWnJUZJuNHaXmLafCewN3tqKGZfsf1TWW5360Wq21RW6wVhuW8gLB4MhsHV5CGFzAvv+7+enWnwfsIyJ8OntA4krdaWtDzvYNJSYfn9bD1X+QrpDf8C</diagram><diagram id="cHXQ7OsaG6UoClwh1egs" name="Model">5VZNc9owEP01HJnBHxhzBEpSpiGlQ6e03IS12ALZC7IIpr++MpaxBZQmaZseemG0T/Ku9N5boYYziLN7QTbRGCnwht2iWcN517Btu9tWvzlwKADfdgogFIwWkFUBU/YdNNjS6I5RSI2FEpFLtjHBAJMEAmlgRAjcm8uWyM2qGxLCBTANCL9EZ4zKqDxFp8LfAwujsrLldYuZmJSL9UnSiFDc1yBn2HAGAlEWozgbAM+5K3npPIzG38RoNH5cRdn2PnS/8mazSHb3kk9ORxCQyFenHlnzZrql29Eh82fLlR8Fs4H+pPVE+E7zpc8qDyWBAncJhTyJ1XD6+4hJmG5IkM/ulWMUFsmY6+klJvKOxIznbukJRrgGtTEsV8VEBDpsq4iSNDql19sBISE70+8Xh7dOiignA8YgxUF9p7P4vj6pdrGrw31lCcfTWFS3QwkSbcPwlLqiWg0029eZ/zIJPG+dhDBx1/206X0cDD88h3mgysk6RCEjDDEhfFih/UqbloqqNQ+IG83oCqQ8aLrJTqKpV1EzL/QKttVmcScCuOGvsuOJCEHeWGdfV08AJ5I9mZv7HSlubbImxURAcyMwgDRF8QYdsWScD5CrWnkBh7bBpzmeSoFrqM349sLxvL/YKF3bbBTLvuyUE1bvFPcPNMrPrWGoM0wlU7fzf6aM1fL/oTTJfL5NXBn77XZn8bn9yXoMe1fvMI+rqv2FGoTyyEYB5LwaWnnbHZYTzfTIeE8tUDRl1WSZRb9KilRqq0U2s4KCa1XPjKG0kKb6poAJJnCmtoYIZ2GiQg7LPEOuK1Ovi56GY0bp8Sq+Zjbzejb+6J5lvrdq8s6lkdwrPrJf7iMVVg+k41ztlekMfwA=</diagram><diagram name="Training" id="74e2e168-ea6b-b213-b513-2b3c1d86103e">5VtLk6M2EP41Pk6KhwFxnIdnc5ip3eykKtmjDAKUFcgR8tjeXx9hJF7yOMyubRjs8oFuvVB3f92tBmb2fbr9xOAqeaYhIjPLCLcz+2Fmid+s+BvhrqSBZZeMmOGwZJk14wX/QJJpSO4ahyhvdeSUEo5XbWZAswwFvMWDjNFNu1tESXvVFYyRxngJING5f+GQJ5JrGkbd8DvCcSKXBo5sWMLge8zoOpPrzSw72v/K5hSquWT/PIEh3TRY9mJm3zNKeXmVbu8RKSSrxFaOe3yjtbpvhjLeZ0D04ylaLm4ezXCXPj/dfY7DP6IbxyuneYVkjdQ+9nfLd0pC+z2iYhZzZt9tEszRywoGRetGWITgJTwlsjmiGX+EKSaFOdwyDIlkSs2bc0FDFkjSEVQI86SaXt+V3OgrYhxtGyy5y0+IpoizneiiWk1Xilya5FySm1q/LpC8pKlaxYTSpuJq7lqu4kKK9j1iBppUUSjMUJKU8YTGNINkUXPv2nKv+zxRupLMfxDnOylKuOb0J3TxpsBzumYBOrIpJWUOWYz4sY7SIxRbPqpAhgjk+LWNzNMrw79qZTijUoa674YD+go3gvEAObyAK4owIfeUULZfwA4hAlEg+Dln9DtqtLgBQMvoRB7KMtoeyrR0F1Xxmi5qfi4P5ZqaHqaIErsvSrxxoeSqleGNK36o+25A5U8GcTZ1p+VYbadVObHhnNZ8irBw+sJiZJHc0WGBcj51VFQnifGgwr2KUO71hYk5LpjoJ+5HXKDkC0M3K0YDlOfCULsKyxO4Ki5lj/+HTF2e+LzmBGfoJGp5BzDszincAoMDY4qncGXeA4SL/dBbxuCu0WFFccbzxsxfCkbDLjyzbRe2Z3Q0W05Z67m6t19Q/RTP/P1Vf/IE+udU7zgd1fsXUL1nDKF6tMX8b3Ft/OZI6luj5aGQiKGInSIysdPGoIL81myrh+0pNW4sZgb8cZjZvHNQAUanMq7ZJTg64Ex2qVdcjqcA58+YHQTC+aGMGVhLW6SV50kMKnqwxMCzriFj9nrXW8ZV/PL0eksDKBPIlLuFlfmBB1YXBsT8KgDRu9Jy8sD2a9rRKy3TAkS3pjICQLhTtP++JRQwrhKKp5dQGvYvBG4ZV1mQr+jhYDLJCovfFybjek6l7vsYTK6wQD88SsAgFYkzowT0rhK4o0IJ0I/hKngsco5TyCdRje/GCnfwx1TgKg7doO+h2x9XjgX0Q/eiuIYcTQwZ3fjgDv6cCgzyWkNdfW7VnutS9DDVZxUuPlr2BfRD+hTdW2/tjKuEAvR3JN72aVMpvGs5wPCezpsiJkBfTIws5IPDmTDO4pnlEl4EawEONy6ubklMGeZJenm0RFFkBQfPjaG7dJ1ToaX7jvbwGbNvTxEtfasrvjUutOjVla8oxjlHAiOG/EruwyfH3ZABBq/E+x/vdY4qux5FQu33LtSMK2Xz9UKNPJFimukB6lksigP9adjpw9MpcNY9hHqDvxOh3G1D3OrT39LPyTUuGvqRKVJl71Do913PhudKlP3htXHogbw0eWXxilHIsKUX9981VQ03+V66t6KDkNK2blSzdFM+cbvlhO1FBLuxcMcOhJx5W9ltfWW0iGQt5UoWJDjOBElQVMxQ6AwHkNxKdorDcO/QD9lWbX2FF219VXwxHHeTRk83nPkBu7HebzeCrL8dL99Jqz/Ptxf/AQ==</diagram><diagram id="GuWJZfnpEX2tFPy7BMl6" name="Calibrating">7Vtde6I4FP41XnYeIPLhZWttbbfudMbu7LQ380SIwBSIG0LV/voNmghIqrajwOjc+JCTD8g55815kxNboBvOrgmceAPsoKClKc6sBS5bmqZ1dPabCuZLgaWBpcAlvrMUqZlg6L8iLlS4NPEdFBcaUowD6k+KQhtHEbJpQQYJwdNiszEOim+dQBeVBEMbBmXpv75DPTELM5P3ke964s2q0VnWhFA05jOJPejgaU4Eei3QJRjT5VM466Ig1Z3Qy6X5/KLfJndW9JJM+p1oeOkOzpaDXb2ny2oKBEX0w0Mno9fpt6d7JVG+aM/ew1M/vLriXZQXGCRcX3yudC4USHASOSgdRG2Bi6nnUzScQDutnTKPYTKPhgGvHuOIXsHQD1JvOSc+DLiQO4baZmVIbF7UWcmBsbcafseZco28IELRLGdnPvNrhENEyZw1EbVKm0+Vu7Gm8PI0cwrd4DIv5xCmMD/kjuiuBs+UzR64vt+h+7KqkcNclxcxoR52cQSDXia9KBoja3OH8YQLfyJK51y/MKH4AwZ6nxVinBAbbZgoXzAoJC6iG9pZy3apFjbalKAAUv+lCPC9mweUoPEVTpngElJYAUjGfhB0cYDJ4gXAgcga20weU4KfUa7GsC00Gh8UO9YadqwydlRNgp32oaBjngp0rB2hozYLO1YJOw8opicJHq1O8JjtL3//DGb9v/ArNl/Cqd1DqB7woJlPv+eeH9mz8knTefEyVZciCnNRiNh8v+cL+W5pOeu3KImOe0Yh86mmw1Bqaas5ljZ3tfQnvWDrhhoagH0betH1nBA4zzWYYD+icW7k+1SQX13WaC3orG0CtnUwrS0dwOYO7GH50ZmXrmb/8fgh7JALIPcEnU0ItlEcs8W7+hCiI8tpy0KIpY2AYVTJv4BRN/9SzZJ9ToWRqbtSMqGShlAytczJcpAqWS/24CR9FPVbQTWC9rO7sOjnhAZ+hA5no19gX5VCRxqTyytbHcjZMyjEJLaDot0ogmSclO6NRi1I4rvlCxJTsKb82TTyZavdrnDZeug4Or65JeFN4t2QPr19hESyoz8G6EjovnT2zQrlWjmGdGHgjwikiIl7MfVDSCUs+UiDuiE5y6+WD2t6I+BRBx8WUWX7lllvFoiMksl66fOJYKi9tq03aj/TB0pJ10cKGSDy4M1lyxu/WxJ3fBy1NCOgqb8zzBhu+nQeuJj41AurJ2/j8VizpeTNMUaGXulxTaXh6enH19fhsPt58HA1077NekB/7J2V7XYE5A1I0srS2SvNApEkr4xcP6aI4UZZ8bh0+8Pv4hxnAFoncRbYESXGwQJQuw5UiHQJT3zwhImo2ZItUYt5sW1psVqCnb4rP9x7SuXXvKFM6Tk/lMa6AXupb5fPUfcf6SqjhGaVZ6XSbfcxHM+BXfdHam1kT6r7em4JvJHwrzjfv7PNtE6z1qzynlbc511yDP6OSqk4Uh0dmTIq3jFMAA9Jxa3GJU6BLHHKY4kIJUKQ6rVgK+O/BIuKs3ih8XPWgGlullWKUVY7s8gVA7IvXo5ZfA8T59695h5M07ToA0UzRjjlkwWbcxF7vxuxYoDG6Qip1XwbBudcHPqOs1g0ZC6XOWWK7sIN4XpjZOkqpIS1tiUOtY9LxNIMFqh3mc5dxarhuo5Yfn+3gxTx3Xlu+eaB40ldbdG1ujln2TS/FwndlL9qcIp45v+gwFUG8fXg+m70z8XFTf91l3+nHK8t6tsPb/psWc4xPas60fWrlIqsMo2y6bTzzxbgDQt1DrgHYMXs74DLu7DZfypB738=</diagram><diagram id="wosXcDLAJT1C6PcZYE3D" name="Predicting">5VrbcpswEP0aP7rDxWB4THxJOtNr3KRJXzIKrEGJQI6Qb/36CiMZMIzjpLVN6heP9khaod09uxK4ZfaixQVDk/Az9YG0DM1ftMx+yzAM1xK/KbDMAN11jQwJGPYllgMj/BskqEl0in1ISgM5pYTjSRn0aByDx0sYYozOy8PGlJRXnaAAKsDIQ6SK/sQ+DzPUMbo5fgk4CNXKuu1mPRFSg+VOkhD5dF6AzEHL7DFKedaKFj0gqfGUXYRysvh+09HORyzR+h8xaV+0M2XD10xZb4FBzN+s+vHx8vr56hYGEC5vFveDczSz5RRthshU2kvulS+VARmdxj6kSvSWeT4PMYfRBHlp71yEjMBCHhHZPaYxH6IIkzRczhhGRIIyMPSOkBHzpGgJyUdJuFa/406lRWbAOCwKfpY7vwAaAWdLMUT2OrbcqQzjtmtKYF4Iio7EwkJAGApEMhCDtfLc2KIh7V1v+/ndj+Fzz47uyfWvNrUul8Nk3DZftj34IpalSBkPaUBjRAY5el72Tj7mE6UTCT4C50tpcDTl9A0ee51bEjplHmyJOrlzjlgAfMs43c4GpmbY6mUGBHE8K1P+bxy27bELDvsCcwFcofS3jzg6AHnGmJAeJZStFjB9BM7YE3jCGX2CQo/tOfAw3iOnXG2DU91ulVO6UcOpvVHKOVlKCZ3vklPquQsu+8agPWHUgyQRsXx4Rlng+J06RjnGg2nbe2SUrm2WKcc6NqWsk6WUMvOLlDI6jaKUUT3UFSglvHCCtUpfn+KbU6zs02XWrsXKtJrFrNpi5WNxbd30nLgsTtKmZN3LhHpA3lOw8ubXKSc4hv355xW0MZ3G0aZ7urRx32lBcisuGyQcR4if2vmuQqeDnu+2pthqTsM0Tg7gnr0Z2+o0LnfpFXseIFXBAvNb0dY+WFK6K/T0U3NpSlgqIRbbLUxKxbtiXz5tJal5R3mbZO+YFvWjpcXaYGhu3XpLxBzLqaq4NKTWmdVTvfqmcgUBTtQaBy15oIui160rea7dNdFeS17l4qVr+tFrXt0Z0iY8PYaLRsBXBsmA1LQld9nPU6o62snK6GdigLDXIu9UWlQpjQOlTzxwprK8jIALS28EiLA1L0dB2ZExTa8LJa9LCBEcxEIkME41pH7DHiJnEo6w76/ySl3Q5WGpbX6mOWphr7x3XgOFgKr9lLOvsl69FjY2tf/jNK3vnKa1RqVpvZqmC+/IKt77T27zldfL+zwRCzH/Nr7qK/zDwBz8AQ==</diagram></mxfile>
+<mxfile host="Electron" modified="2020-12-07T14:30:48.916Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/13.9.5 Chrome/85.0.4183.121 Electron/10.1.5 Safari/537.36" etag="5uIaT5zmQQaQWTK3eDw0" version="13.9.5" type="device" pages="5"><diagram id="xTy2y7PyVUbXui-J8zsi" name="Modelling">7Vhdb9owFP01eeyUD0jhkc9Wa9nQqLR1L5MbG8erkwuOIaG/fg6xSUIq1EqljKoSD77HN9fxuec4JJY3iLIrgRbhBDDhlmvjzPKGlus6juta+c/GmwLpuF4BUMGwTiqBGXsiGrQ1umKYJLVECcAlW9TBAOKYBLKGISEgrafNgddXXSBKGsAsQLyJ/mRYhmYXlyV+TRgNzcqO3y1mImSS9U6SEGFIK5A3sryBAJDFKMoGhOfkGV5Q53t/SfnkhtyiKR9n6Y0zuSiKjV9zyW4LgsTybUvrvSZyY/giWNGnQxAyBAox4qMS7QtYxZjkVR0VlTm3AAsN/iVSbrQW0EqCgkIZcT07h1iOUcR4LqmeYIhrUF/htFT8wj1rbhJYiYAcyGtp6SFByaF6fpGXs1DRj2b0ikBEpNioBEE4kmxdFxnSWqW7vLIfaqBb8or26LteI77SK/U4BcFkGDUaV29LGjJJZgu0JSVV1j56C9ZESJIdJM3MGkfpI8Xt6DgtDbrLCSvmbNlH4tkcVh/fB+2z9EG74YMhkugDWcDzT24Bp0HnO1hA0So2v1Rgm+A+D760TTjMqpPDjY5OYR3/hdbp/FfW8RvW0f/xztY73r533JN7x/30zmHvXJ6ldy4b3vlGUgWc+cNn30At+9QG6jSIngqCWSAZxMkZM90+5WN+8Xg7eZwtPXqHftAphz+d7lf9jlhl2nJ9rlbtP6gBlVsyCiAnqUa8v1yBmbhItvT1VIIiLisnTZU7gVjMYmqqqbstCtYXUXBl4b1GK5ZlvZuJFPBIBsBBKCSGOD9J54zzPQhxRmMVcjLPK+QdYwHiPQ1HDOPtIfyceEp55aclRkm409qLxLT7TmAfUVlud+8dqttUVusZYblvICweDEfB9eQhhcwL7we/n5xZ8H7CMifDp7SOJK3WnrQ872jSUmH5/Ww7V/kK6Y3+AQ==</diagram><diagram id="cHXQ7OsaG6UoClwh1egs" name="Model">5VZNc9owEP01HJnBHxhzBEpSpiGlQ6e03IS12ALZC7IIpr++MpaxBZQmaZseemG0T/Ku9N5boYYziLN7QTbRGCnwht2iWcN517Btu9tWvzlwKADfdgogFIwWkFUBU/YdNNjS6I5RSI2FEpFLtjHBAJMEAmlgRAjcm8uWyM2qGxLCBTANCL9EZ4zKqDxFp8LfAwujsrLldYuZmJSL9UnSiFDc1yBn2HAGAlEWozgbAM+5K3npPIzG38RoNH5cRdn2PnS/8mazSHb3kk9ORxCQyFenHlnzZrql29Eh82fLlR8Fs4H+pPVE+E7zpc8qDyWBAncJhTyJ1XD6+4hJmG5IkM/ulWMUFsmY6+klJvKOxIznbukJRrgGtTEsV8VEBDpsq4iSNDql19sBISE70+8Xh7dOiignA8YgxUF9p7P4vj6pdrGrw31lCcfTWFS3QwkSbcPwlLqiWg0029eZ/zIJPG+dhDBx1/206X0cDD88h3mgysk6RCEjDDEhfFih/UqbloqqNQ+IG83oCqQ8aLrJTqKpV1EzL/QKttVmcScCuOGvsuOJCEHeWGdfV08AJ5I9mZv7HSlubbImxURAcyMwgDRF8QYdsWScD5CrWnkBh7bBpzmeSoFrqM349sLxvL/YKF3bbBTLvuyUE1bvFPcPNMrPrWGoM0wlU7fzf6aM1fL/oTTJfL5NXBn77XZn8bn9yXoMe1fvMI+rqv2FGoTyyEYB5LwaWnnbHZYTzfTIeE8tUDRl1WSZRb9KilRqq0U2s4KCa1XPjKG0kKb6poAJJnCmtoYIZ2GiQg7LPEOuK1Ovi56GY0bp8Sq+Zjbzejb+6J5lvrdq8s6lkdwrPrJf7iMVVg+k41ztlekMfwA=</diagram><diagram name="Training" id="74e2e168-ea6b-b213-b513-2b3c1d86103e">5VtLk6M2EP41Pk6KhwFxnIdnc5ip3eykKtmjDAKUFcgR8tjeXx9hJF7yOMyubRjs8oFuvVB3f92tBmb2fbr9xOAqeaYhIjPLCLcz+2Fmid+s+BvhrqSBZZeMmOGwZJk14wX/QJJpSO4ahyhvdeSUEo5XbWZAswwFvMWDjNFNu1tESXvVFYyRxngJING5f+GQJ5JrGkbd8DvCcSKXBo5sWMLge8zoOpPrzSw72v/K5hSquWT/PIEh3TRY9mJm3zNKeXmVbu8RKSSrxFaOe3yjtbpvhjLeZ0D04ylaLm4ezXCXPj/dfY7DP6IbxyuneYVkjdQ+9nfLd0pC+z2iYhZzZt9tEszRywoGRetGWITgJTwlsjmiGX+EKSaFOdwyDIlkSs2bc0FDFkjSEVQI86SaXt+V3OgrYhxtGyy5y0+IpoizneiiWk1Xilya5FySm1q/LpC8pKlaxYTSpuJq7lqu4kKK9j1iBppUUSjMUJKU8YTGNINkUXPv2nKv+zxRupLMfxDnOylKuOb0J3TxpsBzumYBOrIpJWUOWYz4sY7SIxRbPqpAhgjk+LWNzNMrw79qZTijUoa674YD+go3gvEAObyAK4owIfeUULZfwA4hAlEg+Dln9DtqtLgBQMvoRB7KMtoeyrR0F1Xxmi5qfi4P5ZqaHqaIErsvSrxxoeSqleGNK36o+25A5U8GcTZ1p+VYbadVObHhnNZ8irBw+sJiZJHc0WGBcj51VFQnifGgwr2KUO71hYk5LpjoJ+5HXKDkC0M3K0YDlOfCULsKyxO4Ki5lj/+HTF2e+LzmBGfoJGp5BzDszincAoMDY4qncGXeA4SL/dBbxuCu0WFFccbzxsxfCkbDLjyzbRe2Z3Q0W05Z67m6t19Q/RTP/P1Vf/IE+udU7zgd1fsXUL1nDKF6tMX8b3Ft/OZI6luj5aGQiKGInSIysdPGoIL81myrh+0pNW4sZgb8cZjZvHNQAUanMq7ZJTg64Ex2qVdcjqcA58+YHQTC+aGMGVhLW6SV50kMKnqwxMCzriFj9nrXW8ZV/PL0eksDKBPIlLuFlfmBB1YXBsT8KgDRu9Jy8sD2a9rRKy3TAkS3pjICQLhTtP++JRQwrhKKp5dQGvYvBG4ZV1mQr+jhYDLJCovfFybjek6l7vsYTK6wQD88SsAgFYkzowT0rhK4o0IJ0I/hKngsco5TyCdRje/GCnfwx1TgKg7doO+h2x9XjgX0Q/eiuIYcTQwZ3fjgDv6cCgzyWkNdfW7VnutS9DDVZxUuPlr2BfRD+hTdW2/tjKuEAvR3JN72aVMpvGs5wPCezpsiJkBfTIws5IPDmTDO4pnlEl4EawEONy6ubklMGeZJenm0RFFkBQfPjaG7dJ1ToaX7jvbwGbNvTxEtfasrvjUutOjVla8oxjlHAiOG/EruwyfH3ZABBq/E+x/vdY4qux5FQu33LtSMK2Xz9UKNPJFimukB6lksigP9adjpw9MpcNY9hHqDvxOh3G1D3OrT39LPyTUuGvqRKVJl71Do913PhudKlP3htXHogbw0eWXxilHIsKUX9981VQ03+V66t6KDkNK2blSzdFM+cbvlhO1FBLuxcMcOhJx5W9ltfWW0iGQt5UoWJDjOBElQVMxQ6AwHkNxKdorDcO/QD9lWbX2FF219VXwxHHeTRk83nPkBu7HebzeCrL8dL99Jqz/Ptxf/AQ==</diagram><diagram id="GuWJZfnpEX2tFPy7BMl6" name="Calibrating">7Vtde6I4FP41XnYeIPLhZWttbbfudMbu7LQ380SIwBSIG0LV/voNmghIqrajwOjc+JCTD8g55815kxNboBvOrgmceAPsoKClKc6sBS5bmqZ1dPabCuZLgaWBpcAlvrMUqZlg6L8iLlS4NPEdFBcaUowD6k+KQhtHEbJpQQYJwdNiszEOim+dQBeVBEMbBmXpv75DPTELM5P3ke964s2q0VnWhFA05jOJPejgaU4Eei3QJRjT5VM466Ig1Z3Qy6X5/KLfJndW9JJM+p1oeOkOzpaDXb2ny2oKBEX0w0Mno9fpt6d7JVG+aM/ew1M/vLriXZQXGCRcX3yudC4USHASOSgdRG2Bi6nnUzScQDutnTKPYTKPhgGvHuOIXsHQD1JvOSc+DLiQO4baZmVIbF7UWcmBsbcafseZco28IELRLGdnPvNrhENEyZw1EbVKm0+Vu7Gm8PI0cwrd4DIv5xCmMD/kjuiuBs+UzR64vt+h+7KqkcNclxcxoR52cQSDXia9KBoja3OH8YQLfyJK51y/MKH4AwZ6nxVinBAbbZgoXzAoJC6iG9pZy3apFjbalKAAUv+lCPC9mweUoPEVTpngElJYAUjGfhB0cYDJ4gXAgcga20weU4KfUa7GsC00Gh8UO9YadqwydlRNgp32oaBjngp0rB2hozYLO1YJOw8opicJHq1O8JjtL3//DGb9v/ArNl/Cqd1DqB7woJlPv+eeH9mz8knTefEyVZciCnNRiNh8v+cL+W5pOeu3KImOe0Yh86mmw1Bqaas5ljZ3tfQnvWDrhhoagH0betH1nBA4zzWYYD+icW7k+1SQX13WaC3orG0CtnUwrS0dwOYO7GH50ZmXrmb/8fgh7JALIPcEnU0ItlEcs8W7+hCiI8tpy0KIpY2AYVTJv4BRN/9SzZJ9ToWRqbtSMqGShlAytczJcpAqWS/24CR9FPVbQTWC9rO7sOjnhAZ+hA5no19gX5VCRxqTyytbHcjZMyjEJLaDot0ogmSclO6NRi1I4rvlCxJTsKb82TTyZavdrnDZeug4Or65JeFN4t2QPr19hESyoz8G6EjovnT2zQrlWjmGdGHgjwikiIl7MfVDSCUs+UiDuiE5y6+WD2t6I+BRBx8WUWX7lllvFoiMksl66fOJYKi9tq03aj/TB0pJ10cKGSDy4M1lyxu/WxJ3fBy1NCOgqb8zzBhu+nQeuJj41AurJ2/j8VizpeTNMUaGXulxTaXh6enH19fhsPt58HA1077NekB/7J2V7XYE5A1I0srS2SvNApEkr4xcP6aI4UZZ8bh0+8Pv4hxnAFoncRbYESXGwQJQuw5UiHQJT3zwhImo2ZItUYt5sW1psVqCnb4rP9x7SuXXvKFM6Tk/lMa6AXupb5fPUfcf6SqjhGaVZ6XSbfcxHM+BXfdHam1kT6r7em4JvJHwrzjfv7PNtE6z1qzynlbc511yDP6OSqk4Uh0dmTIq3jFMAA9Jxa3GJU6BLHHKY4kIJUKQ6rVgK+O/BIuKs3ih8XPWgGlullWKUVY7s8gVA7IvXo5ZfA8T59695h5M07ToA0UzRjjlkwWbcxF7vxuxYoDG6Qip1XwbBudcHPqOs1g0ZC6XOWWK7sIN4XpjZOkqpIS1tiUOtY9LxNIMFqh3mc5dxarhuo5Yfn+3gxTx3Xlu+eaB40ldbdG1ujln2TS/FwndlL9qcIp45v+gwFUG8fXg+m70z8XFTf91l3+nHK8t6tsPb/psWc4xPas60fWrlIqsMo2y6bTzzxbgDQt1DrgHYMXs74DLu7DZfypB738=</diagram><diagram id="wosXcDLAJT1C6PcZYE3D" name="Predicting">5VrbcpswEP0aP7rDxWB4THxJOtNr3KRJXzIKrEGJQI6Qb/36CiMZMIzjpLVN6heP9khaod09uxK4ZfaixQVDk/Az9YG0DM1ftMx+yzAM1xK/KbDMAN11jQwJGPYllgMj/BskqEl0in1ISgM5pYTjSRn0aByDx0sYYozOy8PGlJRXnaAAKsDIQ6SK/sQ+DzPUMbo5fgk4CNXKuu1mPRFSg+VOkhD5dF6AzEHL7DFKedaKFj0gqfGUXYRysvh+09HORyzR+h8xaV+0M2XD10xZb4FBzN+s+vHx8vr56hYGEC5vFveDczSz5RRthshU2kvulS+VARmdxj6kSvSWeT4PMYfRBHlp71yEjMBCHhHZPaYxH6IIkzRczhhGRIIyMPSOkBHzpGgJyUdJuFa/406lRWbAOCwKfpY7vwAaAWdLMUT2OrbcqQzjtmtKYF4Iio7EwkJAGApEMhCDtfLc2KIh7V1v+/ndj+Fzz47uyfWvNrUul8Nk3DZftj34IpalSBkPaUBjRAY5el72Tj7mE6UTCT4C50tpcDTl9A0ee51bEjplHmyJOrlzjlgAfMs43c4GpmbY6mUGBHE8K1P+bxy27bELDvsCcwFcofS3jzg6AHnGmJAeJZStFjB9BM7YE3jCGX2CQo/tOfAw3iOnXG2DU91ulVO6UcOpvVHKOVlKCZ3vklPquQsu+8agPWHUgyQRsXx4Rlng+J06RjnGg2nbe2SUrm2WKcc6NqWsk6WUMvOLlDI6jaKUUT3UFSglvHCCtUpfn+KbU6zs02XWrsXKtJrFrNpi5WNxbd30nLgsTtKmZN3LhHpA3lOw8ubXKSc4hv355xW0MZ3G0aZ7urRx32lBcisuGyQcR4if2vmuQqeDnu+2pthqTsM0Tg7gnr0Z2+o0LnfpFXseIFXBAvNb0dY+WFK6K/T0U3NpSlgqIRbbLUxKxbtiXz5tJal5R3mbZO+YFvWjpcXaYGhu3XpLxBzLqaq4NKTWmdVTvfqmcgUBTtQaBy15oIui160rea7dNdFeS17l4qVr+tFrXt0Z0iY8PYaLRsBXBsmA1LQld9nPU6o62snK6GdigLDXIu9UWlQpjQOlTzxwprK8jIALS28EiLA1L0dB2ZExTa8LJa9LCBEcxEIkME41pH7DHiJnEo6w76/ySl3Q5WGpbX6mOWphr7x3XgOFgKr9lLOvsl69FjY2tf/jNK3vnKa1RqVpvZqmC+/IKt77T27zldfL+zwRCzH/Nr7qK/zDwBz8AQ==</diagram></mxfile>

scripts/titanic/titanic_config.yml

@@ -14,5 +14,8 @@ evaluation:
     - accuracy
   shap_analysis: True
 
+calibration:
+  calibrate: True
+
 prediction:
   stage: Production

scripts/titanic/train_titanic.py

@@ -9,6 +9,7 @@
 
 from myautoml.evaluation.binary_classifier import evaluate_binary_classifier
 from myautoml.evaluation.shap import shap_analyse
+from myautoml.calibration import calibrate_model
 from myautoml.utils import load_config
 from myautoml.utils.hyperopt import flatten_params, prep_params
 from myautoml.utils.mlflow import log_sk_model
@@ -54,7 +55,8 @@ def main():
         metrics = {}
         artifacts = {}
 
-        with mlflow.start_run():
+        with mlflow.start_run() as run:
+            run_id = run.info.run_id
             _logger.info("Fitting the preprocessor")
             preprocessor = get_preprocessor()
             preprocessor.fit(x_train, y_train)
@@ -152,7 +154,13 @@ def hyperopt_objective(search_params):
             log_sk_model(model, registered_model_name=None,
                          params=params, tags=tags, metrics=metrics, artifacts=artifacts)
 
-    return (x_train, y_train, x_test, y_test), model, params, tags, metrics, artifacts
+        if config.calibration.calibrate:
+            model_calibration = calibrate_model(run_id, x_test, y_test)
+        else:
+            model_calibration = ()
+
+    model_training = (x_train, y_train, x_test, y_test), model, params, tags, metrics, artifacts
+    return model_training, model_calibration
 
 
 if __name__ == "__main__":

setup.py

@@ -52,7 +52,9 @@
     packages=find_packages(where="src"),
     package_dir={"": "src"},
     install_requires=[
+        'click',
         'joblib',
+        'mlflow',
         'numpy',
         'pandas',
         'pyyaml',
@@ -65,7 +67,7 @@
     ],
     include_package_data=True,
 
-    python_requires='>=3.6',
+    python_requires='>=3.6,<3.9',
     extras_require=extras,
 
     entry_points={

src/myautoml/__about__.py

@@ -6,7 +6,7 @@
 __summary__ = "myautoml is a package which provides a framework to automate machine learning"
 __uri__ = "https://github.com/myautoml/myautoml"
 
-__version__ = "0.3.0"
+__version__ = "0.4.0"
 
 __author__ = "Erik Jan de Vries"
 

src/myautoml/calibration/__init__.py

@@ -0,0 +1,60 @@
+import logging.config
+from pathlib import Path
+import tempfile
+
+import mlflow
+from sklearn.calibration import CalibratedClassifierCV
+
+from myautoml.evaluation.binary_classifier import evaluate_calibration
+from myautoml.utils.mlflow import log_sk_model, get_model
+from myautoml.utils.model import make_pipeline
+
+_logger = logging.getLogger(__file__)
+
+
+def calibrate_model(run_id, x, y):
+    with tempfile.TemporaryDirectory() as td:
+        _logger.debug(f"Creating temporary directory: '{td}'")
+        temp_dir = Path(td)
+
+        params = {}
+        tags = {}
+        metrics = {}
+        artifacts = {}
+
+        _logger.info("Loading the model")
+        model = get_model(run_id, model_path='model')
+
+        with mlflow.start_run(run_id):
+            _logger.info("Preprocessing the training data")
+            preprocessor = model.steps[0][1]
+            x_prep = preprocessor.transform(x)
+
+            # calibrate model
+            calibrated_estimator = CalibratedClassifierCV(model.steps[1][1], cv='prefit')
+            calibrated_estimator.fit(x_prep, y)
+
+            # evaluate the model
+            estimator_metrics, estimator_artifacts = evaluate_calibration(
+                model=calibrated_estimator,
+                data={'test': {'x': x_prep, 'y': y}},
+                temp_dir=temp_dir)
+
+            estimator_params = {}
+            estimator_tags = {'calibrated': True}
+
+            calibrated_model = make_pipeline(preprocessor, calibrated_estimator)
+            params.update({f"estimator_{k}": v for k, v in estimator_params.items()})
+            tags.update({f"estimator_{k}": v for k, v in estimator_tags.items()})
+            metrics.update(estimator_metrics)
+            artifacts.update(estimator_artifacts)
+
+            log_sk_model(calibrated_model,
+                         registered_model_name=None,
+                         params=params,
+                         tags=tags,
+                         metrics=metrics,
+                         artifacts=artifacts,
+                         model_artifact_path='model_calibrated')
+
+    return (x, y), model, params, tags, metrics, artifacts