tests/unintests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<!-- Auto table of contents -->\n",
    "<h1 class='tocIgnore'>Development and unit tests</h1>\n",
    "\n",
    "<p>WARNING: This notebook is becoming obsolete as corresponding unint_*_test.py pytest modules are released</p>\n",
    "\n",
    "**pyaudisam**: Automation of Distance Sampling analyses with [Distance software](http://distancesampling.org/)\n",
    "\n",
    "Copyright (C) 2021 Jean-Philippe Meuret\n",
    "\n",
    "This program is free software: you can redistribute it and/or modify it under the terms\n",
    "of the GNU General Public License as published by the Free Software Foundation,\n",
    "either version 3 of the License, or (at your option) any later version.\n",
    "\n",
    "This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;\n",
    "without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n",
    "See the GNU General Public License for more details.\n",
    "\n",
    "You should have received a copy of the GNU General Public License along with this program.\n",
    "If not, see https://www.gnu.org/licenses/.\n",
    "\n",
    "<div style=\"overflow-y: auto\">\n",
    "  <h2 class='tocIgnore'>Table of contents</h2>\n",
    "  <div id=\"toc\"></div>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "$.getScript('ipython_notebook_toc.js')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import copy\n",
    "import os\n",
    "import pathlib as pl\n",
    "import pickle, lzma\n",
    "\n",
    "import re\n",
    "\n",
    "import concurrent.futures as cofu\n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "from IPython.display import HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activate Warnings as Exception\n",
    "#import warnings\n",
    "#warnings.filterwarnings('error')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Commons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Short identification string for a sample.\n",
    "def sampleAbbrev(sSample):\n",
    "    \n",
    "    abrvSpe = ''.join(word[:4].title() for word in sSample['Espèce'].split(' ')[:2])\n",
    "    \n",
    "    sampAbbrev = '{}-{}-{}-{}'.format(abrvSpe, sSample.Passage.replace('+', ''),\n",
    "                                      sSample.Adulte.replace('+', ''), sSample['Durée'])\n",
    "    \n",
    "    return sampAbbrev\n",
    "\n",
    "# Short identification string for an analysis.\n",
    "def analysisAbbrev(sAnlys):\n",
    "    \n",
    "    # Sample abbreviation\n",
    "    abbrevs = [sampleAbbrev(sAnlys)]\n",
    "\n",
    "    # Model + Parameters abbreviation\n",
    "    abbrevs += [sAnlys['FonctionClé'][:3].lower(), sAnlys['SérieAjust'][:3].lower()]\n",
    "    dTroncAbrv = { 'l': 'TrGche' if 'TrGche' in sAnlys.index else 'TroncGche',\n",
    "                   'r': 'TrDrte' if 'TrDrte' in sAnlys.index else 'TroncDrte',\n",
    "                   'm': 'NbTrches' if 'NbTrches' in sAnlys.index else 'NbTrModel'\n",
    "                                   if 'NbTrModel' in sAnlys.index else  'NbTrchMod',\n",
    "                   'd': 'NbTrDiscr' }\n",
    "    for abrv, name in dTroncAbrv.items():\n",
    "        if name in sAnlys.index and not pd.isnull(sAnlys[name]):\n",
    "            abbrevs.append('{}{}'.format(abrv, sAnlys[name][0].lower() if isinstance(sAnlys[name], str)\n",
    "                                               else int(sAnlys[name])))\n",
    "   \n",
    "    return '-'.join(abbrevs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# I. Unit tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. MCDS.exe detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.path.insert(0, '..')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyaudisam as ads\n",
    "\n",
    "ads.runtime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create temporary directory if not yet done.\n",
    "tmpDir = pl.Path('tmp')\n",
    "tmpDir.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logging configuration.\n",
    "ads.log.configure(handlers=[sys.stdout, tmpDir / 'unintst.log'], reset=True,\n",
    "                  loggers=[dict(name='matplotlib', level=ads.WARNING),\n",
    "                           dict(name='ads', level=ads.INFO),\n",
    "                           # dict(name='ads.dat', level=ads.INFO),\n",
    "                           dict(name='ads.eng', level=ads.INFO2),\n",
    "                           dict(name='ads.opn', level=ads.INFO3),\n",
    "                           dict(name='ads.opr', level=ads.INFO2),\n",
    "                           dict(name='ads.onr', level=ads.INFO2),\n",
    "                           dict(name='ads.anr', level=ads.INFO2),\n",
    "                           dict(name='unintst', level=ads.DEBUG)])\n",
    "\n",
    "logger = ads.logger('unintst', level=ads.DEBUG)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* [13. MCDSAnalysisResultsSet](#13.-MCDSAnalysisResultsSet)\n",
    "* [14. MCDS(Opt)AnalysisResultsSet](#14.-MCDS(Opt)AnalysisResultsSet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. DataSet class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Finish preparing import data set\n",
    "dfPapAlaArv = pd.read_excel('refin/ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.ods')\n",
    "\n",
    "dfPapAlaArv.to_csv('tmp/ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.csv', sep='\\t', index=False)\n",
    "dfPapAlaArv.to_excel('tmp/ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.xls', index=False)  # Need for deprecated module xlwt !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DataSet from multiple sources from various formats (same columns)\n",
    "# => ctor, _csv2df, _fromDataFrame, _fromDataFile, _addComputedColumns, addColumns, renameColumns\n",
    "sources = ['refin/ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.ods',   # Need for module odfpy\n",
    "           'refin/ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.xlsx',   # Need for module openpyxl (or xlrd)\n",
    "           'tmp/ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.xls',  # No need for module xlwt (openpyxl seems to just do it)\n",
    "           'tmp/ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.csv', dfPapAlaArv]\n",
    "\n",
    "def male2bool(s):\n",
    "    return False if pd.isnull(s.MALE) or s.MALE.lower() != 'oui' else True\n",
    "\n",
    "ds = ads.DataSet(sources, importDecFields=['EFFORT', 'DISTANCE', 'NOMBRE'],\n",
    "                 dRenameCols={'NOMBRE': 'INDIVIDUS'}, dComputeCols={'MALE': male2bool},\n",
    "                 sheet='Sheet1', skipRows=None, separator='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# => ctor, _csv2df, _fromDataFrame, _fromDataFile, __len__, columns, empty\n",
    "assert not ds.empty\n",
    "\n",
    "assert len(ds) == len(dfPapAlaArv) * len(sources)\n",
    "\n",
    "assert sorted(ds.columns) == sorted(['ZONE', 'HA', 'POINT', 'ESPECE', 'DISTANCE', 'MALE', 'DATE',\n",
    "                                     'OBSERVATEUR', 'PASSAGE', 'INDIVIDUS', 'EFFORT'])\n",
    "\n",
    "dTypes = {'ZONE': 'object', 'HA': 'int', 'POINT': 'int', 'ESPECE': 'object',\n",
    "          'DISTANCE': 'float', 'MALE': 'bool', 'DATE': 'object', 'OBSERVATEUR': 'object',\n",
    "          'PASSAGE': 'object', 'INDIVIDUS': 'float', 'EFFORT': 'int'}\n",
    "assert all(typ.name.startswith(dTypes[col]) for col, typ in ds.dfData.dtypes.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# => dfData\n",
    "ds.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# => dfSubData, __len__, columns\n",
    "df = ds.dfSubData(columns=['POINT', 'ESPECE', 'DISTANCE', 'INDIVIDUS', 'EFFORT'])\n",
    "\n",
    "assert len(df) == len(dfPapAlaArv) * len(sources)\n",
    "assert df.columns.to_list() == ['POINT', 'ESPECE', 'DISTANCE', 'INDIVIDUS', 'EFFORT']\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# => dfSubData, __len__, columns\n",
    "df = ds.dfSubData(columns=['POINT', 'ESPECE', 'DISTANCE', 'INDIVIDUS', 'EFFORT'], index=range(1, 300, 3))\n",
    "\n",
    "assert len(df) == 100\n",
    "assert df.columns.to_list() == ['POINT', 'ESPECE', 'DISTANCE', 'INDIVIDUS', 'EFFORT']\n",
    "assert df.index.to_list() == list(range(1, 300, 3))\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ctor, _csv2df, _fromDataFrame, _fromDataFile, dfData\n",
    "assert ds.dfData.MALE.value_counts()[True] == ds.dfData.INDIVIDUS.sum() == dfPapAlaArv.NOMBRE.sum() * len(sources)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# => dropColumns, columns, __len__\n",
    "ds.dropColumns(['ZONE', 'HA', 'OBSERVATEUR'])\n",
    "\n",
    "assert len(ds) == len(dfPapAlaArv) * len(sources)\n",
    "assert ds.columns.to_list() == ['POINT', 'ESPECE', 'DISTANCE', 'MALE', 'DATE', 'PASSAGE', 'INDIVIDUS', 'EFFORT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# => dropRows, dfData, __len__\n",
    "ds.dropRows(ds.dfData.DISTANCE.isnull())\n",
    "\n",
    "assert len(ds) == len(dfPapAlaArv[dfPapAlaArv.DISTANCE.notnull()]) * len(sources)\n",
    "assert ds.dfData.MALE.value_counts()[True] == ds.dfData.INDIVIDUS.sum() == dfPapAlaArv.NOMBRE.sum() * len(sources)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# => toExcel, toOpenDoc, toPickle, compareDataFrames\n",
    "closenessThreshold = 15  # => max relative delta = 1e-15\n",
    "subsetCols = ['POINT', 'ESPECE', 'DISTANCE', 'INDIVIDUS', 'EFFORT']\n",
    "filePathName = tmpDir / 'dataset-uni.ods'\n",
    "dfRef = ds.dfSubData(columns=subsetCols).reset_index(drop=True)\n",
    "\n",
    "for fpn in [filePathName, filePathName.with_suffix('.xlsx'), filePathName.with_suffix('.xls'),\n",
    "            filePathName.with_suffix('.pickle'), filePathName.with_suffix('.pickle.xz')]:\n",
    "    \n",
    "    print(fpn.as_posix(), end=' : ')\n",
    "    if fpn.suffix == '.ods':\n",
    "        ds.toOpenDoc(fpn, sheetName='utest', subset=subsetCols, index=False)\n",
    "    elif fpn.suffix in ['.xlsx', '.xls']:\n",
    "        ds.toExcel(fpn, sheetName='utest', subset=subsetCols, index=False)\n",
    "    elif fpn.suffix in ['.pickle', '.xz']:\n",
    "        ds.toPickle(fpn, subset=subsetCols, index=False)\n",
    "    assert fpn.is_file()\n",
    "\n",
    "    if fpn.suffix in ['.ods', '.xlsx', '.xls']:\n",
    "        df = pd.read_excel(fpn, sheet_name='utest')\n",
    "    elif fpn.suffix in ['.pickle', '.xz']:\n",
    "        df = pd.read_pickle(fpn)\n",
    "        df.reset_index(drop=True, inplace=True)\n",
    "    assert ds.compareDataFrames(df.reset_index(), dfRef.reset_index(),\n",
    "                                subsetCols=['POINT', 'DISTANCE', 'INDIVIDUS', 'EFFORT'],\n",
    "                                indexCols=['index'], dropCloser=closenessThreshold, dropNans=True).empty\n",
    "    print('1e-{} comparison OK (df.equals(dfRef) is {}, df.compare(dfRef) {}empty)'\n",
    "          .format(closenessThreshold, df.equals(dfRef), '' if df.compare(dfRef).empty else 'not'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Base function for comparison (test from static hard-coded data, not from loaded DataSets)\n",
    "# => _closeness\n",
    "values = [np.nan, -np.inf,\n",
    "          -1.0e12, -1.0e5, -1.0-1e-5, -1.0, -1.0+1e-5, -1.0e-8,\n",
    "          0.0, 1.0e-8, 1.0, 1.0e5, 1.0e12, np.inf]\n",
    "aClose = np.ndarray(shape=(len(values), len(values)))\n",
    "\n",
    "for r in range(len(values)):\n",
    "    for c in range(len(values)):\n",
    "        try:\n",
    "            aClose[r, c] = ds._closeness(pd.Series([values[r], values[c]]))\n",
    "        except Exception as exc:\n",
    "            print(exc, r, c, values[r], values[c])\n",
    "            raise\n",
    "\n",
    "# Proximité infinie sur la diagonale (sauf pour nan et +/-inf)\n",
    "assert all(np.isnan(values[i]) or np.isinf(values[i]) or np.isinf(aClose[i, i]) for i in range(len(values))), \\\n",
    "       'Error: Inequality on the diagonal'\n",
    "\n",
    "# Pas de proximité infinie ailleurs\n",
    "assert all(r == c or not np.isinf(aClose[r, c]) for r in range(len(values)) for c in range(len(values))), \\\n",
    "       'Error: No equality should be found outside the diagonal'\n",
    "\n",
    "# Bonne proximité uniquement autour de -1\n",
    "whereClose = [i for i in range(len(values)) if abs(values[i] + 1) <= 1.0e-5]\n",
    "assert all(aClose[r, c] > 4 for r in whereClose for c in whereClose), 'Error: Unexpectedly bad closeness around -1'\n",
    "\n",
    "pd.DataFrame(data=aClose, index=values, columns=values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Comparison (from other files data sources, the same as for ResultsSet.compare below, but through DataSet)\n",
    "# => compare, compareDataFrames, _toHashable, _closeness\n",
    "\n",
    "# a. Chargement référence Distance 7 et valeurs à comparer issues de pyaudisam\n",
    "dsDist = ads.DataSet('refin/ACDC2019-Papyrus-ALAARV-TURMER-comp-dist-auto.ods',\n",
    "                     sheet='RefDist73', skipRows=[3], headerRows=[0, 1, 2], indexCols=0)\n",
    "\n",
    "dsAuto = ads.DataSet('refin/ACDC2019-Papyrus-ALAARV-TURMER-comp-dist-auto.ods',\n",
    "                     sheet='ActAuto', skipRows=[3], headerRows=[0, 1, 2], indexCols=0)\n",
    "\n",
    "# b. Colonnes d'index pour la comparaison\n",
    "indexCols = [('sample', 'AnlysNum', 'Value')] \\\n",
    "            + [('sample', col, 'Value') for col in ['Species', 'Periods', 'Prec.', 'Duration']] \\\n",
    "            + [('model', 'Model', 'Value')] \\\n",
    "            + [('parameters', 'left truncation distance', 'Value'),\n",
    "               ('parameters', 'right truncation distance', 'Value'),\n",
    "               ('parameters', 'model fitting distance cut points', 'Value'),\n",
    "               ('parameters', 'distance discretisation cut points', 'Value')]\n",
    "\n",
    "# c. Colonnes à comparer (on retire DeltaDCV et DeltaAIC car ils dépendent des ensembles d'analyses effectuées,\n",
    "#    différents entre la référence et l'exécution auto).\n",
    "subsetCols = [col for col in dsDist.dfData.columns.to_list() \\\n",
    "              if col not in indexCols + [('run output', 'run time', 'Value'),\n",
    "                                         ('density/abundance', 'density of animals', 'Delta Cv'),\n",
    "                                         ('detection probability', 'Delta AIC', 'Value')]]\n",
    "\n",
    "# d. Comparaison \"exacte\" : aucune ligne n'y réussit (majorité d'epsilons dûs à IO ODS)\n",
    "dfRelDiff = dsDist.compare(dsAuto, subsetCols=subsetCols, indexCols=indexCols)\n",
    "assert len(dfRelDiff) == len(dsDist)\n",
    "\n",
    "# e. Comparaison à 10**-16 près : presque toutes les lignes réussissent, sauf 3 (majorité d'epsilons dûs à IO ODS).\n",
    "dfRelDiff = dsDist.compare(dsAuto, subsetCols=subsetCols, indexCols=indexCols, dropCloser=16, dropNans=True)\n",
    "assert len(dfRelDiff) == 3\n",
    "\n",
    "dfRelDiff = dsDist.compare(dsAuto, subsetCols=subsetCols, indexCols=indexCols, dropCloser=5, dropNans=True)\n",
    "assert len(dfRelDiff) == 2\n",
    "\n",
    "dfRelDiff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. SampleDataSet class (and base DataSet)\n",
    "\n",
    "Note: Self-contained, nothing needing to be run before (but 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Excel source (path as simple string)\n",
    "sds = ads.SampleDataSet(source='refin/ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.xlsx',\n",
    "                        decimalFields=['EFFORT', 'DISTANCE', 'NOMBRE'])\n",
    "\n",
    "assert sds.columns.to_list() == ['ZONE', 'HA', 'POINT', 'ESPECE', 'DISTANCE', 'MALE', 'DATE',\n",
    "                                 'OBSERVATEUR', 'PASSAGE', 'NOMBRE', 'EFFORT']\n",
    "assert len(sds) == 256\n",
    "assert sds.dfData.NOMBRE.sum() == 217\n",
    "\n",
    "sds.dfData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Libre / Open Office source (path as simple string)\n",
    "sds = ads.SampleDataSet(source='refin/ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.ods',\n",
    "                        decimalFields=['EFFORT', 'DISTANCE', 'NOMBRE'])\n",
    "\n",
    "assert sds.columns.to_list() == ['ZONE', 'HA', 'POINT', 'ESPECE', 'DISTANCE', 'MALE', 'DATE',\n",
    "                                 'OBSERVATEUR', 'PASSAGE', 'NOMBRE', 'EFFORT']\n",
    "assert len(sds) == 256\n",
    "assert sds.dfData.NOMBRE.sum() == 217\n",
    "\n",
    "sds.dfData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sds.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CSV source with ',' as decimal point (path as pl.Path)\n",
    "sds = ads.SampleDataSet(source=pl.Path('refin/ACDC2019-Papyrus-TURMER-AB-5mn-1dec-dist.txt'),\n",
    "                        decimalFields=['Point transect*Survey effort', 'Observation*Radial distance'])\n",
    "\n",
    "assert not any(sds.dfData[col].dropna().apply(lambda v: isinstance(v, str)).any() for col in sds.decimalFields), \\\n",
    "       'Error: Some strings found in declared decimal fields ... any decimal format issue ?'\n",
    "\n",
    "assert sds.columns.to_list() == ['Region*Label', 'Region*Area', 'Point transect*Label',\n",
    "                                 'Point transect*Survey effort', 'Observation*Radial distance']\n",
    "assert len(sds) == 330\n",
    "assert sds.dfData['Observation*Radial distance'].notnull().sum() == 324\n",
    "\n",
    "sds.dfData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CSV source with '.' as decimal point\n",
    "sds = ads.SampleDataSet(source=pl.Path('refin/ACDC2019-Papyrus-ALAARV-AB-10mn-1dotdec-dist.txt'),\n",
    "                       decimalFields=['Point transect*Survey effort', 'Observation*Radial distance'])\n",
    "\n",
    "assert not any(sds.dfData[col].dropna().apply(lambda v: isinstance(v, str)).any() for col in sds.decimalFields), \\\n",
    "       'Error: Some strings found in declared decimal fields ... any decimal format issue ?'\n",
    "\n",
    "assert sds.columns.to_list() == ['Region*Label', 'Region*Area', 'Point transect*Label',\n",
    "                                 'Point transect*Survey effort', 'Observation*Radial distance']\n",
    "assert len(sds) == 256\n",
    "assert sds.dfData['Observation*Radial distance'].notnull().sum() == 217\n",
    "\n",
    "sds.dfData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DataFrame source.\n",
    "dfData = pd.DataFrame(columns=['Date', 'TrucDec', 'Espece', 'Point', 'Effort', 'Distance'],\n",
    "                      data=[('2019-05-13', 3.5, 'TURMER', 23, 2,   83),\n",
    "                            ('2019-05-15', np.nan, 'TURMER', 23, 2,   27.355),\n",
    "                            ('2019-05-13', 0, 'ALAARV', 29, 2,   56.85),\n",
    "                            ('2019-04-03', 1.325, 'PRUMOD', 53, 1.3,  7.2),\n",
    "                            ('2019-06-01', 2, 'PHICOL', 12, 1,  np.nan),\n",
    "                            ('2019-06-19', np.nan, 'PHICOL', 17, 0.5, np.nan),\n",
    "                           ])\n",
    "dfData['Region'] = 'ACDC'\n",
    "dfData['Surface'] = '2400'\n",
    "\n",
    "sds = ads.SampleDataSet(source=dfData, decimalFields=['Effort', 'Distance', 'TrucDec'])\n",
    "\n",
    "assert not any(sds.dfData[col].dropna().apply(lambda v: isinstance(v, str)).any() for col in sds.decimalFields), \\\n",
    "       'Error: Some strings found in declared decimal fields ... any decimal format issue ?'\n",
    "\n",
    "assert sds.columns.equals(dfData.columns)\n",
    "assert len(sds) == len(dfData)\n",
    "assert sds.dfData.Distance.notnull().sum() == 4\n",
    "\n",
    "sds.dfData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. XXEngine classes\n",
    "\n",
    "Note: Self-contained, nothing needing to be run before (but 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Instance creation et loading of MCDS.exe output stat. specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    eng = ads.MCDSEngine(workDir='tmp/test out') # Simple string path\n",
    "    print('Error: Should have raised an AssertionError !')\n",
    "except AssertionError as exc:\n",
    "    print('Good forbidden chars detection:', exc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    eng = ads.MCDSEngine(workDir=tmpDir / 'test out') # pl.Path path\n",
    "    print('Error: Should have raised an AssertionError !')\n",
    "except AssertionError as exc:\n",
    "    print('Good forbidden chars detection:', exc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The old run method.\n",
    "eng = ads.MCDSEngine(workDir=tmpDir / 'mcds-out', runMethod='os.system')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runDir = eng.setupRunFolder(runPrefix='uni') # Unit tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Generate input data file for MCDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A short dataset.\n",
    "dfData = pd.DataFrame(columns=['Date', 'TrucDec', 'Espece', 'Point', 'Effort', 'Distance'],\n",
    "                      data=[('2019-05-13', 3.5, 'TURMER', 23, 2,   83),\n",
    "                            ('2019-05-15', np.nan, 'TURMER', 23, 2,   27.355),\n",
    "                            ('2019-05-13', 0, 'ALAARV', 29, 2,   56.85),\n",
    "                            ('2019-04-03', 1.325, 'PRUMOD', 53, 1.3,  7.2),\n",
    "                            ('2019-06-01', 2, 'PHICOL', 12, 1,  np.nan),\n",
    "                            ('2019-06-19', np.nan, 'PHICOL', 17, 0.5, np.nan),\n",
    "                           ])\n",
    "dfData['Region'] = 'ACDC'\n",
    "dfData['Surface'] = '2400'\n",
    "\n",
    "sds = ads.SampleDataSet(source=dfData, decimalFields=['Effort', 'Distance', 'TrucDec'])\n",
    "\n",
    "sds.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataFileName = eng.buildDataFile(sampleDataSet=sds, runDir=runDir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Compute sample stats for MCDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sSmpStats = eng.computeSampleStats(sds)\n",
    "sSmpStats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert all(sSmpStats.index == eng.MIStatSampCols)\n",
    "assert all(sSmpStats.values == [4, 7.2, 83.0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. Generate input command file for MCDS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cmdFileName = eng.buildCmdFile(estimKeyFn='HNORMAL', estimAdjustFn='COSINE', estimCriterion='AIC', cvInterval=95,\n",
    "                               runDir=runDir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "### e. Low level analysis execution (_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Debug mode\n",
    "runStatus, startTime, elapsedTime = \\\n",
    "    eng._run(eng.ExeFilePathName, cmdFileName, forReal=False, method=eng.runMethod)\n",
    "\n",
    "dict(runStatus=runStatus, startTime=startTime, elapsedTime=elapsedTime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Real mode\n",
    "runStatus, startTime, engElapsedTime = \\\n",
    "    eng._run(eng.ExeFilePathName, cmdFileName, forReal=True, method=eng.runMethod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Real mode\n",
    "runStatus, startTime, engElapsedTime = \\\n",
    "    eng._run(eng.ExeFilePathName, cmdFileName, forReal=True, method=eng.runMethod)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Timeout\n",
    "runStatus, startTime, engElapsedTime = \\\n",
    "    eng._run(eng.ExeFilePathName, cmdFileName, forReal=True, method='subprocess.run', timeOut=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng = ads.MCDSEngine(workDir=tmpDir / 'mcds-out')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%timeit -r 5 -n 10\n",
    "\n",
    "# Performance measures : method='os.system'\n",
    "# * Python 3.8 + Core i5 8365U (4 HT cores, 1.6-4.1GHz, cache  6Mb, bus 4GT/s) + SSD 256Gb NVME + RAM 16Gb \"Optimal performance power scheme\"\n",
    "#   - 2021-01-06: 132 ms ± 1.47 ms per loop (mean ± std. dev. of 5 runs, 10 loops each)\n",
    "#   - 2021-10-02: 134 ms ± 6.31 ms per loop (mean ± std. dev. of 5 runs, 10 loops each)\n",
    "# * Python 3.8 + Core i7 10850H (6 HT cores, 2.7-5.1GHz, cache 12Mb, bus 8GT/s) + SSD 512Gb NVME + RAM 32Gb \"Optimal performance power scheme\"\n",
    "#   - 2023-11-02: 42.2 ms ± 822 µs per loop (mean ± std. dev. of 5 runs, 10 loops each)\n",
    "\n",
    "runStatus, startTime, engElapsedTime = \\\n",
    "    eng._run(eng.ExeFilePathName, cmdFileName, forReal=True, method='os.system')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%timeit -r 5 -n 10\n",
    "\n",
    "# Performance measures : method='subprocess.run'\n",
    "# * Python 3.8 + Core i5 8365U (4 HT cores, 1.6-4.1GHz, cache  6Mb, bus 4GT/s) + SSD 256Gb NVME + RAM 16Gb \"Optimal performance power scheme\"\n",
    "#   - 2021-01-06: 191 ms ± 3.75 ms per loop (mean ± std. dev. of 5 runs, 10 loops each) => os.system faster by 60-75ms\n",
    "#   - 2021-10-02: 211 ms ± 19.7 ms per loop (mean ± std. dev. of 5 runs, 10 loops each) => os.system faster by ~80ms\n",
    "# * Python 3.8 + Core i7 10850H (6 HT cores, 2.7-5.1GHz, cache 12Mb, bus 8GT/s) + SSD 512Gb NVME + RAM 32Gb \"Optimal performance power scheme\"\n",
    "#   - 2023-11-02: 46.6 ms ± 727 µs per loop (mean ± std. dev. of 5 runs, 10 loops each) => os.system faster by ~5ms only (see above)\n",
    "runStatus, startTime, engElapsedTime = \\\n",
    "    eng._run(eng.ExeFilePathName, cmdFileName, forReal=True, method='subprocess.run')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Conclusion**\n",
    "\n",
    "Windows 10, 4-core i5-8350U, PCI-e SSD, \"optimal performances\" power scheme:\n",
    "* 2021-01-06 : os.system systematically faster by 60-75ms\n",
    "* 2021-10-02 : os.system systematically faster by ~80ms (128 ms ± 2.49 ms _vs_ 207 ms ± 13.5 ms)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "### f. High level analysis execution  (via executor), debug mode\n",
    "\n",
    "(generate cmd and data input files, but no call to executable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A real life (reduced) dataset\n",
    "sds = ads.SampleDataSet(source=pl.Path('refin') / 'ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.xlsx',\n",
    "                        decimalFields=['EFFORT', 'DISTANCE', 'NOMBRE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Asynchronous model, even if no parallelism involved : submitAnalysis() returns a \"future\" object\n",
    "# (see module concurrent)\n",
    "futRun = eng.submitAnalysis(sds, realRun=False, runPrefix='int',\n",
    "                            estimKeyFn='UNIFORM', estimAdjustFn='POLY',\n",
    "                            estimCriterion='AIC', cvInterval=95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get run output from future object\n",
    "runCode, startTime, elapsedTime, runDir, sResults = futRun.result()\n",
    "\n",
    "assert runCode == ads.MCDSEngine.RCNotRun, 'Should have NOT run (run code = 0)'\n",
    "\n",
    "dict(runCode=runCode, runDir=runDir, startTime=startTime, elapsedTime=elapsedTime, sResults=sResults)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "runCode, startTime, elapsedTime, runDir, sResults = \\\n",
    "    eng.submitAnalysis(sds, realRun=False, runPrefix='int',\n",
    "                       estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95).result()\n",
    "\n",
    "assert runCode == ads.MCDSEngine.RCNotRun, 'Should have NOT run (run code = 0)'\n",
    "\n",
    "dict(runCode=runCode, runDir=runDir, startTime=startTime, elapsedTime=elapsedTime, sResults=sResults)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "### g. High level analysis execution  (via executor), real mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "KRunCheckErrMsg = {ads.MCDSEngine.RCOK: 'Oh, oh, should have run smoothly and successfully !',\n",
    "                   ads.MCDSEngine.RCWarnings: 'Oh, oh, should have run smoothly (even if with warnings) !',\n",
    "                   ads.MCDSEngine.RCTimedOut: 'Oh, oh, should have timed-out !'}\n",
    "\n",
    "def checkEngineAnalysisRun(sampleDataSet, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           minDist=None, maxDist=None, fitDistCuts=None, discrDistCuts=None,\n",
    "                           runMethod='os.system', timeOut=None, expectRunCode=ads.MCDSEngine.RCOK):\n",
    "    \n",
    "    # Need for an async. executor for time limit checking with os.system run method.\n",
    "    exor = None if runMethod != 'os.system' or timeOut is None else ads.Executor(threads=1)\n",
    "        \n",
    "    # Engine\n",
    "    eng = ads.MCDSEngine(executor=exor, workDir=tmpDir / 'mcds-out',\n",
    "                         runMethod=runMethod, timeOut=timeOut)\n",
    "    \n",
    "    # Run analysis and get results\n",
    "    fut = eng.submitAnalysis(sampleDataSet, realRun=True, runPrefix='int',\n",
    "                             estimKeyFn=estimKeyFn, estimAdjustFn=estimAdjustFn,\n",
    "                             estimCriterion=estimCriterion, cvInterval=cvInterval,\n",
    "                             minDist=minDist, maxDist=maxDist,\n",
    "                             fitDistCuts=fitDistCuts, discrDistCuts=discrDistCuts)\n",
    "    \n",
    "    try:\n",
    "        if timeOut is not None:\n",
    "            startTime = pd.Timestamp.now()  # In case of cofu.TimeoutError\n",
    "        runCode, startTime, elapsedTime, runDir, sResults = fut.result(timeout=timeOut)\n",
    "    except cofu.TimeoutError:\n",
    "        logger.info('MCDS Analysis run timed-out after {}s'.format(timeOut))\n",
    "        runCode, startTime, elapsedTime, runDir, sResults = \\\n",
    "            eng.RCTimedOut, startTime, timeOut, None, None\n",
    "\n",
    "    # Check status\n",
    "    assert runCode == expectRunCode, KRunCheckErrMsg.get(expectRunCode, 'Oh, oh, unexpected expected run code ;-)')\n",
    "    \n",
    "    # Done\n",
    "    eng.shutdown()\n",
    "    if exor:\n",
    "        exor.shutdown()\n",
    "    \n",
    "    return runCode, startTime, elapsedTime, runDir, sResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No time limit\n",
    "runCode, startTime, elapsedTime, runDir, sResults = \\\n",
    "    checkEngineAnalysisRun(sds, estimKeyFn='NEXPON', estimAdjustFn='COSINE', estimCriterion='AIC', cvInterval=95,\n",
    "                           runMethod='os.system', timeOut=None, expectRunCode=ads.MCDSEngine.RCWarnings)\n",
    "\n",
    "runCode, startTime, elapsedTime, runDir, sResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some time limit, but too long to stop analysis.\n",
    "runCode, startTime, elapsedTime, runDir, sResults = \\\n",
    "    checkEngineAnalysisRun(sds, estimKeyFn='HNORMAL', estimAdjustFn='COSINE', estimCriterion='AIC', cvInterval=95,\n",
    "                           runMethod='os.system', timeOut=3, expectRunCode=ads.MCDSEngine.RCWarnings)\n",
    "\n",
    "runCode, startTime, elapsedTime, runDir, sResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Too short time limit => analysis time-out (but MCDS goes on to its end : no kill done by executor)\n",
    "runCode, startTime, elapsedTime, runDir, sResults = \\\n",
    "    checkEngineAnalysisRun(sds, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           runMethod='os.system', timeOut=0.1, expectRunCode=ads.MCDSEngine.RCTimedOut)\n",
    "\n",
    "logger.info('Look: MCDS was not killed, it has gone to its end, whereas the analysis has timed-out')\n",
    "\n",
    "runCode, startTime, elapsedTime, runDir, sResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No time limit\n",
    "runCode, startTime, elapsedTime, runDir, sResults = \\\n",
    "    checkEngineAnalysisRun(sds, estimKeyFn='NEXPON', estimAdjustFn='COSINE', estimCriterion='AIC', cvInterval=95,\n",
    "                           runMethod='subprocess.run', timeOut=None, expectRunCode=ads.MCDSEngine.RCWarnings)\n",
    "\n",
    "runCode, startTime, elapsedTime, runDir, sResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some time limit, but too long to stop analysis.\n",
    "runCode, startTime, elapsedTime, runDir, sResults = \\\n",
    "    checkEngineAnalysisRun(sds, estimKeyFn='HNORMAL', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           runMethod='subprocess.run', timeOut=3, expectRunCode=ads.MCDSEngine.RCErrors)\n",
    "\n",
    "runCode, startTime, elapsedTime, runDir, sResults"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Too short time limit => analysis time-out (but MCDS goes on to its end : no kill done by executor)\n",
    "runCode, startTime, elapsedTime, runDir, sResults = \\\n",
    "    checkEngineAnalysisRun(sds, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           runMethod='subprocess.run', timeOut=0.1, expectRunCode=ads.MCDSEngine.RCTimedOut)\n",
    "\n",
    "logger.info('Look: MCDS was actually killed on time-out')\n",
    "\n",
    "runCode, startTime, elapsedTime, runDir, sResults"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### h. Generate input data files for interactive Distance software\n",
    "\n",
    "('point transect' mode only as for now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tgtDir = pl.Path(eng.workDir, 'distance-in')\n",
    "tgtDir.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Case 1: Point transect with radial distance, no extra fields, no clustering.\n",
    "distDataFileName = \\\n",
    "    eng.buildDistanceDataFile(sds, tgtFilePathName=tgtDir / 'import-data-noextra.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Point transect with radial distance, with extra fields, no clustering.\n",
    "distDataFileName = \\\n",
    "    eng.buildDistanceDataFile(sds, tgtFilePathName=tgtDir / 'import-data-withextra.txt',\n",
    "                              withExtraFields=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Case 2: Point transect with radial distance, no extra fields, with clustering.\n",
    "eng = ads.MCDSEngine(workDir=tmpDir / 'mcds-out', clustering=True)\n",
    "\n",
    "# Add cluster data to the data set\n",
    "dfData['Nombre'] = [1, 2, 1, 1, 2, 3]\n",
    "sds = ads.SampleDataSet(source=dfData, decimalFields=['Effort', 'Distance', 'TrucDec'])\n",
    "\n",
    "# Generate distance file\n",
    "tgtDir = pl.Path(eng.workDir, 'distance-in')\n",
    "tgtDir.mkdir(exist_ok=True)\n",
    "\n",
    "distDataFileName = \\\n",
    "    eng.buildDistanceDataFile(sds, tgtFilePathName=tgtDir / 'import-data-clusters.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distDataFileName.as_posix()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### i. TODO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test lower level code and update unint_engine_test.py accordingly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# decodeStats\n",
    "#sStats = eng.decodeStats(eng.workDir / 'alarv-l15r170f12d6-z0k4foj1')\n",
    "#sStats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# decodePlots\n",
    "# loadDataFile\n",
    "# ..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. MCDSAnalysis class\n",
    "\n",
    "Note: Self-contained, nothing needing to be run before (but 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def checkAnalysisRun(sampleDataSet, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                     minDist=None, maxDist=None, fitDistCuts=None, discrDistCuts=None,\n",
    "                     runMethod='os.system', timeOut=None, expectStatus=ads.MCDSEngine.RCOK):\n",
    "    \n",
    "    # Need for a parallel executor for time limit checking with os.system run method.\n",
    "    exor = None if runMethod != 'os.system' or timeOut is None else ads.Executor(threads=1)\n",
    "        \n",
    "    # Engine\n",
    "    eng = ads.MCDSEngine(executor=exor, workDir=tmpDir / 'mcds-out',\n",
    "                         runMethod=runMethod, timeOut=timeOut)\n",
    "    \n",
    "    # Analysis\n",
    "    anlys = ads.MCDSAnalysis(engine=eng, sampleDataSet=sampleDataSet, name='anlys', logData=True,\n",
    "                             estimKeyFn=estimKeyFn, estimAdjustFn=estimAdjustFn,\n",
    "                             estimCriterion=estimCriterion, cvInterval=cvInterval,\n",
    "                             minDist=minDist, maxDist=maxDist, fitDistCuts=fitDistCuts, discrDistCuts=discrDistCuts)\n",
    "\n",
    "    # Run\n",
    "    anlys.submit()\n",
    "    \n",
    "    # Get result\n",
    "    sResult = anlys.getResults()\n",
    "\n",
    "    # Check status\n",
    "    sts = sResult[('run output', 'run status', 'Value')]\n",
    "    assert sts == expectStatus, KRunCheckErrMsg.get(expectStatus, 'Oh, oh, unexpected expected status ;-)')\n",
    "    \n",
    "    # Done\n",
    "    eng.shutdown()\n",
    "    if exor:\n",
    "        exor.shutdown()\n",
    "    \n",
    "    return sResult"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Dataset to work with ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A real life (reduced) dataset\n",
    "sds = ads.SampleDataSet(source=pl.Path('refin') / 'ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.xlsx',\n",
    "                        decimalFields=['EFFORT', 'DISTANCE', 'NOMBRE'])\n",
    "\n",
    "sds.dfData.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Engine 'os.system' RunMethod and run time limit management"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No time limit\n",
    "sResult = checkAnalysisRun(sds, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           minDist=None, maxDist=None, fitDistCuts=None, discrDistCuts=None,\n",
    "                           runMethod='os.system', timeOut=None, expectStatus=ads.MCDSEngine.RCWarnings)\n",
    "\n",
    "sResult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some time limit, but too long to stop analysis.\n",
    "sResult = checkAnalysisRun(sds, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           minDist=None, maxDist=None, fitDistCuts=None, discrDistCuts=None,\n",
    "                           runMethod='os.system', timeOut=5, expectStatus=ads.MCDSEngine.RCWarnings)\n",
    "\n",
    "sResult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Too short time limit => analysis time-out\n",
    "sResult = checkAnalysisRun(sds, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           minDist=None, maxDist=None, fitDistCuts=None, discrDistCuts=None,\n",
    "                           runMethod='os.system', timeOut=0.01, expectStatus=ads.MCDSEngine.RCTimedOut)\n",
    "\n",
    "sResult"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Engine 'subprocess.run' RunMethod and run time limit management"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No time limit\n",
    "sResult = checkAnalysisRun(sds, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           minDist=None, maxDist=None, fitDistCuts=None, discrDistCuts=None,\n",
    "                           runMethod='os.system', timeOut=None, expectStatus=ads.MCDSEngine.RCWarnings)\n",
    "\n",
    "sResult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some time limit, but too long to stop analysis.\n",
    "sResult = checkAnalysisRun(sds, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           minDist=None, maxDist=None, fitDistCuts=None, discrDistCuts=None,\n",
    "                           runMethod='os.system', timeOut=5, expectStatus=ads.MCDSEngine.RCWarnings)\n",
    "\n",
    "sResult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Too short time limit => analysis time-out\n",
    "sResult = checkAnalysisRun(sds, estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95,\n",
    "                           minDist=None, maxDist=None, fitDistCuts=None, discrDistCuts=None,\n",
    "                           runMethod='os.system', timeOut=0.01, expectStatus=ads.MCDSEngine.RCTimedOut)\n",
    "\n",
    "sResult"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. Performance tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RunMethod='subprocess.run'\n",
    "eng = ads.MCDSEngine(workDir=tmpDir / 'mcds-out', runMethod='subprocess.run')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%timeit -r 5 -n 10\n",
    "\n",
    "# Core i5  8365U (4 HT cores, 1.6-4.1GHz, cache  6Mb, bus 4GT/s) + SSD 256Gb NVME + RAM 16Gb \"Optimal performance power scheme\"\n",
    "# * 2020-01-06: 347 ms ± 8.71 ms per loop (mean ± std. dev. of 5 runs, 10 loops each)\n",
    "# * 2021-10-02: 326 ms ± 2.71 ms per loop (mean ± std. dev. of 5 runs, 10 loops each)\n",
    "\n",
    "# Core i7 10850H (6 HT cores, 2.7-5.1GHz, cache 12Mb, bus 8GT/s) + SSD 512Gb NVME + RAM 32Gb \"Optimal performance power scheme\"\n",
    "# * 20213-11-02: 169 ms ± 1.94 ms per loop (mean ± std. dev. of 5 runs, 10 loops each)\n",
    "runCode, startTime, elapsedTime, runDir, sRes = \\\n",
    "    eng.submitAnalysis(sds, realRun=True, runPrefix='int',\n",
    "                       estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95).result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RunMethod='os.system'\n",
    "eng = ads.MCDSEngine(workDir=tmpDir / 'mcds-out', runMethod='os.system')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%timeit -r 5 -n 10\n",
    "\n",
    "# Python 3.8 + Core i5  8365U (4 HT cores, 1.6-4.1GHz, cache  6Mb, bus 4GT/s) + SSD 256Gb NVME + RAM 16Gb \"Optimal performance power scheme\"\n",
    "# * 2020-01-06: 272 ms ± 7.57 ms per loop (mean ± std. dev. of 5 runs, 10 loops each)\n",
    "# * 2021-10-02: 268 ms ± 20.4 ms per loop (mean ± std. dev. of 5 runs, 10 loops each)\n",
    "\n",
    "# Python 3.8 + Core i7 10850H (6 HT cores, 2.7-5.1GHz, cache 12Mb, bus 8GT/s) + SSD 512Gb NVME + RAM 32Gb \"Optimal performance power scheme\"\n",
    "# * 20213-11-02: 171 ms ± 5.89 ms per loop (mean ± std. dev. of 5 runs, 10 loops each)\n",
    "runCode, startTime, elapsedTime, runDir, sRes = \\\n",
    "    eng.submitAnalysis(sds, realRun=True, runPrefix='int',\n",
    "                       estimKeyFn='UNIFORM', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95).result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng.shutdown()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. AnalysisResultsSet and ResultsSet classes\n",
    "\n",
    "Note: Self-contained, nothing needing to be run before (but 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. ResultsSet class with specialised postComputeColumns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A specialized results set for the tests = with extra. post-computed columns : Delta AIC\n",
    "class TestAnalysisResultsSet(ads.analyser.AnalysisResultsSet):\n",
    "    \n",
    "    def __init__(self, miCustomCols=None, dfCustomColTrans=None,\n",
    "                       dComputedCols=None, dfComputedColTrans=None):\n",
    "        \n",
    "        # Initialise base.\n",
    "        super().__init__(ads.MCDSAnalysis, miCustomCols, dfCustomColTrans, dComputedCols, dfComputedColTrans)\n",
    "        \n",
    "    # Post-computations.\n",
    "    def postComputeColumns(self):\n",
    "        \n",
    "        # Compute Delta AIC (AIC - min(group)) per { species, sample, precision, duration } group.\n",
    "        # a. Minimum AIC per group\n",
    "        aicColInd = ('detection probability', 'AIC value', 'Value')\n",
    "        aicGroupColInds = [('sample', 'species', 'Value'), ('sample', 'periods', 'Value'),\n",
    "                           ('sample', 'duration', 'Value'), ('variant', 'precision', 'Value')]\n",
    "        df2Join = self._dfData.groupby(aicGroupColInds)[[aicColInd]].min()\n",
    "        \n",
    "        # b. Rename computed columns to target\n",
    "        deltaAicColInd = ('detection probability', 'Delta AIC', 'Value')\n",
    "        df2Join.columns = pd.MultiIndex.from_tuples([deltaAicColInd])\n",
    "        \n",
    "        # c. Join the column to the target data-frame\n",
    "        self._dfData = self._dfData.join(df2Join, on=aicGroupColInds)\n",
    "        \n",
    "        # d. Compute delta-AIC in-place\n",
    "        self._dfData[deltaAicColInd] = self._dfData[aicColInd] - self._dfData[deltaAicColInd]\n",
    "\n",
    "# Results object construction\n",
    "miCustCols = pd.MultiIndex.from_tuples([('id', 'index', 'Value'),\n",
    "                                        ('sample', 'species', 'Value'),\n",
    "                                        ('sample', 'periods', 'Value'),\n",
    "                                        ('sample', 'duration', 'Value'),\n",
    "                                        ('variant', 'precision', 'Value')])\n",
    "dfCustColTrans = \\\n",
    "    pd.DataFrame(index=miCustCols,\n",
    "                 data=dict(en=['index', 'species', 'periods', 'duration', 'precision'],\n",
    "                           fr=['numéro', 'espèce', 'périodes', 'durée', 'précision']))\n",
    "\n",
    "dCompCols = {('detection probability', 'Delta AIC', 'Value'): \n",
    "             len(ads.MCDSEngine.statSampCols()) + len(ads.MCDSAnalysis.MIRunColumns) + 11} # Right before AIC\n",
    "dfCompColTrans = \\\n",
    "    pd.DataFrame(index=dCompCols.keys(),\n",
    "                 data=dict(en=['Delta AIC'], fr=['Delta AIC']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                            dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Some getters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# empty\n",
    "assert rs.empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len\n",
    "assert len(rs) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# index\n",
    "assert len(rs.index) == 0\n",
    "assert rs.index.to_list() == []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# columns\n",
    "assert len(rs.columns) == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Append result rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# append\n",
    "sHead = pd.Series(index=miCustCols, data=list(range(len(miCustCols))))\n",
    "\n",
    "miResCols = ads.MCDSEngine.statSampCols().append(ads.MCDSAnalysis.MIRunColumns).append(ads.MCDSEngine.statModCols())\n",
    "\n",
    "sResult = pd.Series(index=miResCols, data=list(range(len(miResCols)))) # Fictive data, never mind !\n",
    "rs.append(sResult, sCustomHead=sHead)\n",
    "\n",
    "sResult = pd.Series(index=miResCols, data=list(range(1, len(miResCols) + 1))) # Fictive data, never mind !\n",
    "rs.append(sResult, sCustomHead=sHead)\n",
    "\n",
    "sResult = pd.Series(index=miResCols, data=list(range(2, len(miResCols) + 2))) # Fictive data, never mind !\n",
    "rs.append(sResult, sCustomHead=sHead)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. Some getters again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfRawData (no post-computed columns)\n",
    "dfRaw = rs.dfRawData\n",
    "dfRaw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# columns (Beware: rs.columns does trigger computation of ... computed columns !)\n",
    "assert len(rs._dfData.columns) == len(dfRaw.columns) and len(dfRaw.columns) == 113\n",
    "rawCols = rs._dfData.columns.to_list()\n",
    "rawCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# columns\n",
    "assert len(rs.columns) == 114  # The proof here !\n",
    "postCols = rs.columns.to_list()\n",
    "postCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check added == compute column\n",
    "assert set(rs.columns.to_list()) - set(dfRaw.columns.to_list()) == { ('detection probability', 'Delta AIC', 'Value') }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfData (post-computations already done, never mind)\n",
    "dfPost = rs.dfData\n",
    "dfPost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# index\n",
    "assert len(rs.index) == 3\n",
    "assert rs.index.to_list() == [0, 1, 2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e. Getters: dfSubData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [('id', 'index', 'Value'), ('sample', 'species', 'Value'),\n",
    "           ('sample', 'periods', 'Value'), ('sample', 'duration', 'Value'),\n",
    "           ('detection probability', 'Delta AIC', 'Value')]\n",
    "index = [0, 2]\n",
    "\n",
    "dfSub = rs.dfSubData(index=index, columns=columns)\n",
    "assert len(dfSub) == 2\n",
    "assert dfSub.index.to_list() == index\n",
    "assert dfSub.columns.to_list() == columns\n",
    "dfSub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### f. Getters: Translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfTransData\n",
    "dfTrans = rs.dfTransData('fr')\n",
    "assert len(dfPost.columns) == len(dfTrans.columns)\n",
    "dfTrans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTrans.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTrSub = rs.dfTransData('en', index=index, columns=columns)\n",
    "assert len(dfTrSub) == 2\n",
    "assert dfTrSub.index.to_list() == index\n",
    "assert dfTrSub.columns.to_list() == ['index', 'species', 'periods', 'duration', 'Delta AIC']\n",
    "dfTrSub"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### g. Specs management"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.updateSpecs(d=dict(a=1, b=2), df=pd.DataFrame([dict(a=3, b=4), dict(a=7, b=9, v=90)]), reset=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.updateSpecs(l=[9, -9], s=pd.Series(dict(e=3, f=5), name='serie'))\n",
    "rs.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    rs.updateSpecs(l=[8, -8, 0])\n",
    "    assert False, \"Error: Should have refused to overwite already existing 'l'\"\n",
    "except AssertionError:\n",
    "    print('Good: Refused to overwrite existing spec if not explicitly authorised to')\n",
    "\n",
    "assert rs.specs['l'] == [9, -9]\n",
    "\n",
    "rs.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.updateSpecs(**dict(l = [7, -7, 77]), overwrite=True)\n",
    "\n",
    "print('Good: Accepted to overwrite existing spec if explicitly authorised to')\n",
    "\n",
    "assert rs.specs['l'] == [7, -7, 77]\n",
    "\n",
    "rs.specs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### h. Imports and exports"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### i. Exports (with specs)\n",
    "\n",
    "(see imports tests below for exported content checks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.toExcel('tmp/results-set-uni.xlsx', sheetName='utest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.toExcel('tmp/results-set-uni.xls', sheetName='utest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.toOpenDoc('tmp/results-set-uni.ods', sheetName='utest')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.toPickle('tmp/results-set-uni.pickle.xz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rs.toPickle('tmp/results-set-uni.pickle')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### ii. Imports with explicit format (with specs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A. XLSX Format\n",
    "rs1 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs1.fromExcel('tmp/results-set-uni.xlsx', sheetName='utest')\n",
    "\n",
    "rs1.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs1.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs1.specs['d'], dict) and rs1.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs1.specs['df'], pd.DataFrame) and rs1.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs1.specs['l'], list) and rs1.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs1.specs['s'], pd.Series) and rs1.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs1.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs1.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B. XLS Format\n",
    "rs2 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs2.fromExcel('tmp/results-set-uni.xls', sheetName='utest')\n",
    "\n",
    "rs2.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs2.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs2.specs['d'], dict) and rs2.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs2.specs['df'], pd.DataFrame) and rs2.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs2.specs['l'], list) and rs2.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs2.specs['s'], pd.Series) and rs2.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs2.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs2.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C. Format ODS\n",
    "rs3 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs3.fromOpenDoc('tmp/results-set-uni.ods', sheetName='utest')\n",
    "\n",
    "rs3.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs3.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs3.specs['d'], dict) and rs3.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs3.specs['df'], pd.DataFrame) and rs3.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs3.specs['l'], list) and rs3.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs3.specs['s'], pd.Series) and rs3.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs3.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs3.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# D. Format pickle comprimé\n",
    "rs4 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs4.fromPickle('tmp/results-set-uni.pickle.xz')\n",
    "\n",
    "rs4.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs4.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs4.specs['d'], dict) and rs4.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs4.specs['df'], pd.DataFrame) and rs4.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs4.specs['l'], list) and rs4.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs4.specs['s'], pd.Series) and rs4.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs4.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs4.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# E. Format pickle non comprimé\n",
    "rs5 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs5.fromPickle('tmp/results-set-uni.pickle')\n",
    "\n",
    "rs5.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs5.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs5.specs['d'], dict) and rs5.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs5.specs['df'], pd.DataFrame) and rs5.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs5.specs['l'], list) and rs5.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs5.specs['s'], pd.Series) and rs5.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs5.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs5.specs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### iii. Imports with auto-detected format (with specs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A. XLSX Format\n",
    "rs1 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs1.fromFile('tmp/results-set-uni.xlsx', sheetName='utest')\n",
    "\n",
    "rs1.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs1.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs1.specs['d'], dict) and rs1.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs1.specs['df'], pd.DataFrame) and rs1.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs1.specs['l'], list) and rs1.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs1.specs['s'], pd.Series) and rs1.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs1.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs1.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# B. XLS Format\n",
    "rs2 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs2.fromFile('tmp/results-set-uni.xls', sheetName='utest')\n",
    "\n",
    "rs2.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs2.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs2.specs['d'], dict) and rs2.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs2.specs['df'], pd.DataFrame) and rs2.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs2.specs['l'], list) and rs2.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs2.specs['s'], pd.Series) and rs2.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs2.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs2.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# C. Format ODS\n",
    "rs3 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs3.fromFile('tmp/results-set-uni.ods', sheetName='utest')\n",
    "\n",
    "rs3.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs3.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs3.specs['d'], dict) and rs3.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs3.specs['df'], pd.DataFrame) and rs3.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs3.specs['l'], list) and rs3.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs3.specs['s'], pd.Series) and rs3.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs3.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs3.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# D. Format pickle comprimé\n",
    "rs4 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs4.fromFile('tmp/results-set-uni.pickle.xz')\n",
    "\n",
    "rs4.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs4.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs4.specs['d'], dict) and rs4.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs4.specs['df'], pd.DataFrame) and rs4.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs4.specs['l'], list) and rs4.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs4.specs['s'], pd.Series) and rs4.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs4.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs4.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# E. Format pickle non comprimé\n",
    "rs5 = TestAnalysisResultsSet(miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                             dComputedCols=dCompCols, dfComputedColTrans=dfCompColTrans)\n",
    "\n",
    "rs5.fromFile('tmp/results-set-uni.pickle')\n",
    "\n",
    "rs5.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data\n",
    "assert rs5.dfData.equals(rs.dfData)  # == fails on NaNs in same places ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs\n",
    "assert isinstance(rs5.specs['d'], dict) and rs5.specs['d'] == rs.specs['d']\n",
    "assert isinstance(rs5.specs['df'], pd.DataFrame) and rs5.specs['df'].equals(rs.specs['df'])  # == fails on NaNs in same places\n",
    "assert isinstance(rs5.specs['l'], list) and rs5.specs['l'] == rs.specs['l']\n",
    "assert isinstance(rs5.specs['s'], pd.Series) and rs5.specs['s'].name == rs.specs['s'].name \\\n",
    "       and rs5.specs['s'].equals(rs.specs['s'])  #  == fails on NaNs in same places\n",
    "\n",
    "rs5.specs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### iv. Imports with default values for missing columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO\n",
    "# How ?\n",
    "# For each file format,\n",
    "# - read target file (written above) with pandas API (not ResultsSet one)\n",
    "# - remove some columns\n",
    "# - overwrite target file with pandas API\n",
    "# - load target file with ResultsSet API, specifying default valeus for the missing columns\n",
    "# - check that results is OK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### i. Comparison\n",
    "\n",
    "Note: Self-contained, nothing needing to be run before (but 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Objets MCDSAnalysisResultsSet et chargement depuis fichiers.\n",
    "modelIdCols = ['Model']\n",
    "modelParamCols = ['LTrunc', 'RTrunc', 'FitDistCuts', 'DiscrDistCuts']\n",
    "sampleIdCols = ['Species', 'Periods', 'Prec.', 'Duration']\n",
    "caseIdCols = ['AnlysNum', 'SampNum'] + sampleIdCols + modelIdCols\n",
    "\n",
    "sampCols = [('sample', col, 'Value') for col in sampleIdCols]\n",
    "miSampCols = pd.MultiIndex.from_tuples(sampCols)\n",
    "\n",
    "custCols = [('sample', 'AnlysNum', 'Value'), ('sample', 'SampNum', 'Value')] + sampCols + [('model', 'Model', 'Value')]\n",
    "miCustCols = pd.MultiIndex.from_tuples(custCols)\n",
    "\n",
    "dfCustColTrans = \\\n",
    "    pd.DataFrame(index=miCustCols,\n",
    "                 data=dict(en=caseIdCols, \n",
    "                           fr=['NumAnlys', 'NumSamp', 'Espèce', 'Périodes', 'Préc.', 'Durée', 'Modèle']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Référence (obtenue avec Distance 7.3)\n",
    "rsDist = ads.MCDSAnalysisResultsSet(miSampleCols=miSampCols, sampleIndCol=('sample', 'SampNum', 'Value'),\n",
    "                                    miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                                    distanceUnit='Meter', areaUnit='Hectare',\n",
    "                                    surveyType='Point', distanceType='Radial', clustering=False)\n",
    "\n",
    "rsDist.fromFile('refin/ACDC2019-Papyrus-ALAARV-TURMER-comp-dist-auto.ods', sheetName='RefDist73',\n",
    "                postComputed=True) # Avoid recomputations, some columns are now missing, files are old actually !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Résultat obtenu via pyaudisam.\n",
    "rsAuto = ads.MCDSAnalysisResultsSet(miSampleCols=miSampCols, sampleIndCol=('sample', 'SampNum', 'Value'),\n",
    "                                    miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                                    distanceUnit='Meter', areaUnit='Hectare',\n",
    "                                    surveyType='Point', distanceType='Radial', clustering=False)\n",
    "\n",
    "rsAuto.fromFile('refin/ACDC2019-Papyrus-ALAARV-TURMER-comp-dist-auto.ods', sheetName='ActAuto',\n",
    "                postComputed=True) # Avoid recomputations, some columns are now missing, files are old actually !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes d'index\n",
    "indexCols = custCols + [('parameters', 'left truncation distance', 'Value'),\n",
    "                        ('parameters', 'right truncation distance', 'Value'),\n",
    "                        ('parameters', 'model fitting distance cut points', 'Value'),\n",
    "                        ('parameters', 'distance discretisation cut points', 'Value')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes à comparer (on retire DeltaDCV et DeltaAIC car ils dépendent des ensembles d'analyses effectuées,\n",
    "#                      différents entre la référence et l'exécution auto, et une colonne string : comparaison non implémentée).\n",
    "subsetCols = [col for col in rsDist.columns.to_list() \\\n",
    "              if col not in indexCols + [('run output', 'run time', 'Value'),\n",
    "                                         ('density/abundance', 'density of animals', 'Delta Cv'),\n",
    "                                         ('detection probability', 'Delta AIC', 'Value')]]\n",
    "#subsetCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRelDiff = rsDist.compare(rsAuto, subsetCols=subsetCols, indexCols=indexCols)\n",
    "assert len(dfRelDiff.columns) == 21\n",
    "assert len(dfRelDiff) == len(rsDist)\n",
    "dfRelDiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRelDiff = rsDist.compare(rsAuto, subsetCols=subsetCols, indexCols=indexCols, dropCloser=16, dropNans=False)\n",
    "assert len(dfRelDiff.columns) == 21\n",
    "assert len(dfRelDiff) == 8\n",
    "dfRelDiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRelDiff = rsDist.compare(rsAuto, subsetCols=subsetCols, indexCols=indexCols, dropCloser=16, dropNans=True)\n",
    "assert len(dfRelDiff.columns) == 21\n",
    "assert len(dfRelDiff) == 3\n",
    "dfRelDiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfRelDiff = rsDist.compare(rsAuto, subsetCols=subsetCols, indexCols=indexCols, dropCloser=5, dropNans=True)\n",
    "assert len(dfRelDiff.columns) == 21\n",
    "assert len(dfRelDiff) == 2\n",
    "dfRelDiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop also closer columns\n",
    "dfRelDiff = rsDist.compare(rsAuto, subsetCols=subsetCols, indexCols=indexCols, dropCloser=5, dropNans=True, dropCloserCols=True)\n",
    "assert len(dfRelDiff.columns) == 19\n",
    "dfRelDiff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### j. Post-computations\n",
    "\n",
    "Note: Self-contained, nothing needing to be run before (but 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MCDSAnalysisResultsSet object + loading from file\n",
    "modelIdCols = ['Model']\n",
    "modelParamCols = ['LTrunc', 'RTrunc', 'FitDistCuts', 'DiscrDistCuts']\n",
    "sampleIdCols = ['Species', 'Periods', 'Prec.', 'Duration']\n",
    "caseIdCols = ['AnlysNum', 'SampNum'] + sampleIdCols + modelIdCols\n",
    "\n",
    "sampCols = [('sample', col, 'Value') for col in sampleIdCols]\n",
    "miSampCols = pd.MultiIndex.from_tuples(sampCols)\n",
    "\n",
    "custCols = [('sample', 'AnlysNum', 'Value'), ('sample', 'SampNum', 'Value')] + sampCols + [('model', 'Model', 'Value')]\n",
    "miCustCols = pd.MultiIndex.from_tuples(custCols)\n",
    "\n",
    "dfCustColTrans = \\\n",
    "    pd.DataFrame(index=miCustCols,\n",
    "                 data=dict(en=caseIdCols, \n",
    "                           fr=['NumAnlys', 'NumSamp', 'Espèce', 'Périodes', 'Préc.', 'Durée', 'Modèle']))\n",
    "\n",
    "rsAuto = ads.MCDSAnalysisResultsSet(miSampleCols=miSampCols, sampleIndCol=('sample', 'SampNum', 'Value'),\n",
    "                                    miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                                    distanceUnit='Meter', areaUnit='Hectare',\n",
    "                                    surveyType='Point', distanceType='Radial', clustering=False)\n",
    "\n",
    "rsAuto.fromFile('refin/ACDC2019-Papyrus-ALAARV-TURMER-comp-dist-auto.ods', sheetName='ActAuto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Trigger post-computations\n",
    "rsAuto.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load reference from file\n",
    "rsAutoRef = ads.MCDSAnalysisResultsSet(miSampleCols=miSampCols, sampleIndCol=('sample', 'SampNum', 'Value'),\n",
    "                                       miCustomCols=miCustCols, dfCustomColTrans=dfCustColTrans,\n",
    "                                       distanceUnit='Meter', areaUnit='Hectare',\n",
    "                                       surveyType='Point', distanceType='Radial', clustering=False)\n",
    "\n",
    "rsAutoRef.fromFile('refout/ACDC2019-Papyrus-ALAARV-TURMER-resultats-postcomp.ods')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Comparison of loaded results to reference\n",
    "# a. Index columns\n",
    "indexCols = custCols + [('parameters', 'left truncation distance', 'Value'),\n",
    "                        ('parameters', 'right truncation distance', 'Value'),\n",
    "                        ('parameters', 'model fitting distance cut points', 'Value'),\n",
    "                        ('parameters', 'distance discretisation cut points', 'Value'),\n",
    "                        ('parameters', 'estimator key function', 'Value'),\n",
    "                        ('parameters', 'estimator adjustment series', 'Value'),\n",
    "                        ('parameters', 'estimator selection criterion', 'Value')]\n",
    "\n",
    "# b. Colonnes to compare : we ignore ...\n",
    "# * DeltaDCV et DeltaAIC because they depend on the whole set of analyses actually done to get the results,\n",
    "#   that is possibly different sets in the 2 cases.\n",
    "# * other string-typed columns (comparison not implemented)\n",
    "subsetCols = [col for col in rsAutoRef.columns.to_list() \\\n",
    "              if col not in indexCols + [('run output', 'run time', 'Value'), ('run output', 'run folder', 'Value'),\n",
    "                                         ('density/abundance', 'density of animals', 'Delta Cv'),\n",
    "                                         ('detection probability', 'Delta AIC', 'Value'),\n",
    "                                         ('detection probability', 'key function type', 'Value'),\n",
    "                                         ('detection probability', 'adjustment series type', 'Value')]]\n",
    "\n",
    "# c. Comparison\n",
    "dfRelDiff = rsAuto.compare(rsAutoRef, subsetCols=subsetCols, indexCols=indexCols, dropCloser=15)\n",
    "assert len(dfRelDiff) == 0\n",
    "dfRelDiff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Class FieldDataSet (and base DataSet)\n",
    "\n",
    "Note: For real unit tests of DataSet, see `visionat` module, which defines the same class (have to be the same: check it !)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Load data sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObs = pd.read_csv('refin/ACDC2019-Naturalist-extrait-ObsBrutesAvecDist.txt', sep='\\t', decimal=',')\n",
    "dfObs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countCols =  ['nMalAd10', 'nAutAd10', 'nMalAd5', 'nAutAd5']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sCounts = dfObs[countCols].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dfObs), sCounts.to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(dfObs) == 724\n",
    "assert not any(sCounts - pd.Series({'nMalAd10': 613, 'nAutAd10': 192, 'nMalAd5': 326, 'nAutAd5': 102}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. FieldDataSet._separateMultiCategoryCounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "dfObsMonoCat_ = ads.FieldDataSet._separateMultiCategoryCounts(dfObs, countCols)\n",
    "len(dfObsMonoCat_), dfObsMonoCat_[countCols].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = dfObs[countCols].apply(lambda s: len(s[s > 0]), axis='columns')\n",
    "\n",
    "print(len(s), s.value_counts().to_dict())\n",
    "\n",
    "assert len(s) - len(s[s < 1]) + sum((i-1)*len(s[s == i]) for i in range(1, s.max()+1)) == len(dfObsMonoCat_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(dfObsMonoCat_) == 1125\n",
    "assert not any(dfObsMonoCat_[countCols].sum() - sCounts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsMonoCat_.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Categorise sightings\n",
    "\n",
    "Needed for adding absence data below\n",
    "\n",
    "(no more counts - by the way, all 0 or 1 - => only catgories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Should not see any sightings with all null counts\n",
    "assert dfObsMonoCat_[~dfObsMonoCat_[countCols].any(axis='columns')].empty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count2AdultCat(sCounts):\n",
    "    return 'm' if 'Mal' in sCounts[sCounts > 0].index[0] else 'a'\n",
    "dfObsMonoCat_['Adulte'] = dfObsMonoCat_[countCols].apply(count2AdultCat, axis='columns')\n",
    "\n",
    "def count2DurationCat(sCounts):\n",
    "    return '5mn' if '5' in sCounts[sCounts > 0].index[0] else '10mn'\n",
    "dfObsMonoCat_['Durée'] = dfObsMonoCat_[countCols].apply(count2DurationCat, axis='columns')\n",
    "\n",
    "dfObsMonoCat_.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. FieldDataSet._individualiseMonoCategoryCounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "dfObsIndiv_ = ads.FieldDataSet._individualiseMonoCategoryCounts(dfObsMonoCat_, countCols)\n",
    "len(dfObsIndiv_), dfObsIndiv_[countCols].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(dfObsIndiv_) == 1233\n",
    "assert not any(dfObsIndiv_[countCols].sum() - sCounts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndiv_.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e. FieldDataSet.monoCategorise\n",
    "\n",
    "(combines a, b, c and d above in one function : the one to use actually !)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First, load FieldDataSet from dfObs\n",
    "fds = ads.FieldDataSet(source=dfObs, countCols=countCols,\n",
    "                       addMonoCatCols={ 'Adulte': count2AdultCat, 'Durée': count2DurationCat })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsMonoCat = fds.monoCategorise()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsMonoCat.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (dfObsMonoCat == dfObsMonoCat_).all().all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### f. FieldDataSet.individualise\n",
    "(combines a, b, c and d above in one function : the one to use actually !)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndiv = fds.individualise()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndiv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (dfObsIndiv == dfObsIndiv_).all().all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Second, try from source CSV file\n",
    "fds = ads.FieldDataSet(source='refin/ACDC2019-Naturalist-extrait-ObsBrutesAvecDist.txt',\n",
    "                       importDecFields=['distMem'], countCols=countCols,\n",
    "                       addMonoCatCols={ 'Adulte': count2AdultCat, 'Durée': count2DurationCat })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndiv = fds.individualise()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndiv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (dfObsIndiv == dfObsIndiv_).all().all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### g. Drop now unneeded count columns\n",
    "\n",
    "(only 0 or 1 inside + columns Adulte and Duree to explain what a 1 means)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No more need for count cols then (only 0 or 1 inside + columns Adulte and Duree to explain what a 1 means)\n",
    "dfObsIndiv.drop(columns=countCols, inplace=True)\n",
    "dfObsIndiv.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Class MonoCategoryDataSet (and base DataSet)\n",
    "\n",
    "Note: For real unit tests of DataSet, see `visionat` module, which defines the same class (have to be the same: check it !)\n",
    "\n",
    "Note: Run 4 above before."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Extract transect info\n",
    "\n",
    "(assuming that each transect x pass gave at least 1 sighting, otherwise the effort will be wrong)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transectPlaceCol = 'Point'\n",
    "transectPlaceCols = [transectPlaceCol]\n",
    "passIdCol = 'Passage'\n",
    "effortCol = 'Effort'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfTransPassEffort = ads.MonoCategoryDataSet._extractTransects(dfObsIndiv, transectPlaceCols=transectPlaceCols,\n",
    "                                                              passIdCol=passIdCol,\n",
    "                                                              effortCol=effortCol, effortConstVal=1)\n",
    "dfTransPassEffort"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(dfTransPassEffort) == 41 \\\n",
    "       and len(dfTransPassEffort[dfTransPassEffort.Passage == 'a']) == 21 \\\n",
    "       and len(dfTransPassEffort[dfTransPassEffort.Passage == 'b']) == 20 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Select sighting from 1 sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define sample columns\n",
    "sampleCols = ['Passage', 'Adulte', 'Durée']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndiv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select 1 sample\n",
    "espece = 'Sylvia atricapilla'\n",
    "passage = 'a'\n",
    "adulte = 'm'\n",
    "duree = '10mn'\n",
    "#dfObsIndivSmpl = dfObsIndiv[(dfObsIndiv.Passage == passage) & (dfObsIndiv.Adulte == adulte) \\\n",
    "#                            & (dfObsIndiv.Duree == duree) & (dfObsIndiv.Espece == espece)]\n",
    "\n",
    "dfObsIndivSmpl, dfTrPassEffSmpl = \\\n",
    "    ads.MonoCategoryDataSet._selectSampleSightings(dSample={ 'Passage': passage, 'Adulte': adulte,\n",
    "                                                            'Durée': duree, 'Espèce': espece },\n",
    "                                                  dfAllSights=dfObsIndiv, dfAllEffort=dfTransPassEffort,\n",
    "                                                  transectPlaceCols=['Point'], passIdCol='Passage',\n",
    "                                                  effortCol='Effort')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(dfObsIndivSmpl) == 36 and dfObsIndivSmpl[transectPlaceCol].nunique() == 18\n",
    "assert len(dfTrPassEffSmpl) == 21 and dfTrPassEffSmpl.reset_index()[transectPlaceCol].nunique() == len(dfTrPassEffSmpl)\n",
    "assert len(dfTrPassEffSmpl[dfTrPassEffSmpl.Effort != 1]) == 0 # 1 seul passage, et sur tous les points sans exception"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Add abscence sightings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndivSmpl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "dfObsIndivAbscSmpl = ads.MonoCategoryDataSet._addAbsenceSightings(dfObsIndivSmpl, sampleCols, dfTrPassEffSmpl)\n",
    "len(dfObsIndivAbscSmpl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check for no change in sample columns\n",
    "assert list(dfObsIndivAbscSmpl.columns) == list(dfObsIndivSmpl.columns)\n",
    "\n",
    "# Check for number of added rows\n",
    "assert len(dfObsIndivAbscSmpl) == 39 # 36 sightings + 3 missings transects\n",
    "\n",
    "# Check for final number of transects\n",
    "assert dfObsIndivAbscSmpl[dfTrPassEffSmpl.index.name].nunique() == 21\n",
    "\n",
    "# Check for no change in sample identification\n",
    "assert list(dfObsIndivAbscSmpl['Espèce'].unique()) == [espece, None] # None for absence sightings !\n",
    "assert list(dfObsIndivAbscSmpl.Passage.unique()) == [passage]\n",
    "assert list(dfObsIndivAbscSmpl.Adulte.unique()) == [adulte]\n",
    "assert list(dfObsIndivAbscSmpl['Durée'].unique()) == [duree]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted(dfObsIndiv['Espèce'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Performance test\n",
    "print('Espèce      Passage  Adulte Durée NbDonnées')\n",
    "\n",
    "for espece in ['Sylvia atricapilla', 'Alauda arvensis', 'Sylvia communis', 'Phylloscopus collybita']: \n",
    "    \n",
    "    for passage in ['a', 'b', 'a+b']: \n",
    "\n",
    "        for adulte in ['m', 'a', 'm+a']:\n",
    "\n",
    "            for duree in ['5mn', '10mn']:\n",
    "\n",
    "                passages = passage.split('+')\n",
    "                adultes = adulte.split('+')\n",
    "                #dfObsIndivSmpl = dfObsIndiv[dfObsIndiv.Passage.isin(passages) & dfObsIndiv.Adulte.isin(adultes) \\\n",
    "                #                            & (dfObsIndiv.Duree == duree) & (dfObsIndiv.Espece == espece)]\n",
    "                dfObsIndivSmpl, dfTrPassEffSmpl = \\\n",
    "                    ads.MonoCategoryDataSet._selectSampleSightings(dSample={ 'Passage': passage, 'Adulte': adulte,\n",
    "                                                                            'Durée': duree, 'Espèce': espece },\n",
    "                                                                  dfAllSights=dfObsIndiv,\n",
    "                                                                  dfAllEffort=dfTransPassEffort,\n",
    "                                                                  transectPlaceCols=['Point'], passIdCol='Passage', \n",
    "                                                                  effortCol='Effort')\n",
    "\n",
    "                try:\n",
    "                    print(espece, passage, adulte, duree, ':', len(dfObsIndivSmpl), '=> ', end='')\n",
    "                    dfObsIndivAbscSmpl_ = \\\n",
    "                        ads.MonoCategoryDataSet._addAbsenceSightings(dfObsIndivSmpl, sampleCols, dfTrPassEffSmpl)\n",
    "                    print(len(dfObsIndivAbscSmpl_))\n",
    "                except Exception as e:\n",
    "                    print(e)\n",
    "                    \n",
    "print('Should give around 1s on a Core i7 8850H (6 HT cores, 2.6-4.3GHz, cache 9Mb) + NVME SSD')\n",
    "print('Should give around 1s on a Core i5 8365U (4 HT cores, 1.6-4.1GHz, cache 6Mb) + NVME SSD')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. ads.MonoCategoryDataSet._addSurveyAreaInfo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dSurveyArea = dict(Zone='ACDC', Surface='2400')\n",
    "\n",
    "dfObsIndivAbscSmpl = ads.MonoCategoryDataSet._addSurveyAreaInfo(dfObsIndivAbscSmpl, dSurveyArea=dSurveyArea)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfObsIndivAbscSmpl.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e. MonoCategoryDataSet.sampleDataSet\n",
    "\n",
    "(combines a, b, c and d above in one function : the one to use actually, of course !)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mds = ads.MonoCategoryDataSet(dfObsIndiv, dSurveyArea=dSurveyArea, sampleDecFields=['Effort', 'distMem'],\n",
    "                             transectPlaceCols=transectPlaceCols, passIdCol=passIdCol,\n",
    "                             effortCol=effortCol, effortConstVal=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sds = mds.sampleDataSet(sSampleSpecs=pd.Series({ 'Passage': passage, 'Adulte': adulte, \n",
    "                                                 'Durée': duree, 'Espèce': espece }))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sds.dfData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Abstract class Analyser\n",
    "\n",
    "Note: Run 4 above before."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Generate implicit partial variant combination table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Nombre d'individus par espèce, pour voir quelles espèces on va analyser\n",
    "dfIndivCounts = dfObsIndiv.loc[dfObsIndiv.Adulte == 'm', ['Espèce', 'Adulte']].groupby('Espèce').count()\n",
    "\n",
    "dfIndivCounts.rename(columns=dict(Adulte='Mâles'), inplace=True)\n",
    "dfIndivCounts.sort_values(by='Mâles', ascending=False, inplace=True)\n",
    "\n",
    "dfIndivCounts[dfIndivCounts['Mâles'] >= 20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nMaxMal10 = 30\n",
    "varEspeces = list(dfIndivCounts[dfIndivCounts['Mâles'] >= nMaxMal10].index) # 1 variante par espèce\n",
    "\n",
    "varPassages = [''] # Tous les passages ensemble => 1 seule variante\n",
    "varAdultes = ['m', 'm+a'] # Les mâles, et ensuite les mâles et autres adultes (=> 2 variantes)\n",
    "varDurees = ['5mn', '10mn'] # 5 1ères mn, ou toutes les 10 => 2 variantes\n",
    "\n",
    "dfImplSampSpecs = ads.Analyser.implicitPartialVariantSpecs({ 'Espèces':varEspeces, 'Passages': varPassages,\n",
    "                                                               'Adultes': varAdultes, 'Durées': varDurees })\n",
    "dfImplSampSpecs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Explicit partial variant combination generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfExplSampSpecs = ads.Analyser.explicitPartialVariantSpecs(dfImplSampSpecs)\n",
    "dfExplSampSpecs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Direct explicitation of all variants\n",
    "\n",
    "from user specs (implicit and explict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "userVariantSpecs = 'refin/ACDC2019-Naturalist-extrait-SpecsAnalyses.xlsx'\n",
    "\n",
    "if False: # Both method MUST work, but this one needs more code :-)\n",
    "    userVariantSpecs = pd.read_excel(userVariantSpecs, sheet_name=None)\n",
    "    print('sheets:', ', '.join(userVariantSpecs.keys()))\n",
    "\n",
    "userVariantSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfFinalExplSpecs = ads.Analyser.explicitVariantSpecs(userVariantSpecs, ignore=['Params3_expl'],\n",
    "                                                     varIndCol='IndAnlys',\n",
    "                                                     #convertCols={ 'Durée': int }, # float 'cause of Excel\n",
    "                                                     computedCols=dict(AbrevAnlys=analysisAbbrev))\n",
    "\n",
    "dfFinalExplSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Just to see by eye\n",
    "dfFinalExplSpecs.to_excel('tmp/tools-unitests-final-expl-specs.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Computational checks\n",
    "if isinstance(userVariantSpecs, dict):\n",
    "    ddfUserVariantSpecs = userVariantSpecs\n",
    "else:\n",
    "    ddfUserVariantSpecs = pd.read_excel('refin/ACDC2019-Naturalist-extrait-SpecsAnalyses.xlsx', sheet_name=None)\n",
    "\n",
    "nEch1Vars = 1\n",
    "df = ddfUserVariantSpecs['Echant1_impl']\n",
    "for col in df.columns:\n",
    "    nEch1Vars *= len(df[col].dropna())\n",
    "    \n",
    "nEch2Vars = 1\n",
    "df = ddfUserVariantSpecs['Echant2_impl']\n",
    "for col in df.columns:\n",
    "    nEch2Vars *= len(df[col].dropna())\n",
    "    \n",
    "nModVars = 1\n",
    "df = ddfUserVariantSpecs['Modl_impl']\n",
    "for col in df.columns:\n",
    "    nModVars *= len(df[col].dropna())\n",
    "\n",
    "nEch1ParWithVars = \\\n",
    "  len(ddfUserVariantSpecs['Params1_expl'].drop_duplicates(subset=ddfUserVariantSpecs['Echant1_impl'].columns))\n",
    "\n",
    "nEch1Pars = len(ddfUserVariantSpecs['Params1_expl'])\n",
    "\n",
    "nEch2ParWithVars = \\\n",
    "  len(ddfUserVariantSpecs['Params2_expl'].drop_duplicates(subset=ddfUserVariantSpecs['Echant2_impl'].columns))\n",
    "\n",
    "nEch2Pars = len(ddfUserVariantSpecs['Params2_expl'])\n",
    "\n",
    "nExpdVars = nModVars * (nEch1Pars + nEch1Vars - nEch1ParWithVars + nEch2Pars + nEch2Vars - nEch2ParWithVars)\n",
    "assert len(dfFinalExplSpecs) == nExpdVars\n",
    "\n",
    "nModVars, nEch1Pars, nEch1Vars, nEch1ParWithVars, nEch2Pars, nEch2Vars, nEch2ParWithVars, nExpdVars"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. Abstract class DSAnalyser\n",
    "\n",
    "Note: Run 6 above before."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. userSpec2ParamNames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "IntSpecEstimKeyFn = 'EstimKeyFn'\n",
    "IntSpecEstimAdjustFn = 'EstimAdjustFn'\n",
    "IntSpecEstimCriterion = 'EstimCriterion'\n",
    "IntSpecCVInterval = 'CvInterval'\n",
    "IntSpecMinDist = 'MinDist' # Left truncation distance\n",
    "IntSpecMaxDist = 'MaxDist' # Right truncation distance\n",
    "IntSpecFitDistCuts = 'FitDistCuts'\n",
    "IntSpecDiscrDistCuts = 'DiscrDistCuts'\n",
    "\n",
    "int2UserSpecREs = \\\n",
    "  { IntSpecEstimKeyFn:     ['ke[a-z]*[\\.\\-_ ]*f', 'f[o]?n[a-z]*[\\.\\-_ ]*cl'],\n",
    "    IntSpecEstimAdjustFn:  ['ad[a-z]*[\\.\\-_ ]*s', 's[éa-z]*[\\.\\-_ ]*aj'],\n",
    "    IntSpecEstimCriterion: ['crit[èa-z]*[\\.\\-_ ]*'],\n",
    "    IntSpecCVInterval:     ['conf[a-z]*[\\.\\-_ ]*[a-z]*[\\.\\-_ ]*int',\n",
    "                            'in[o]?n[a-z]*[\\.\\-_ ]*conf'],\n",
    "    IntSpecMinDist:        ['min[a-z]*[\\.\\-_ ]*d', 'd[a-z]*[\\.\\-_ ]*min',\n",
    "                            'tr[a-z]*[\\.\\-_ ]*ga', 'tr[a-z]*[\\.\\-_ ]*gc', 'le[a-z]*[\\.\\-_ ]*tr'],\n",
    "    IntSpecMaxDist:        ['max[a-z]*[\\.\\-_ ]*d', 'd[a-z]*[\\.\\-_ ]*max',\n",
    "                            'tr[a-z]*[\\.\\-_ ]*dr', 'tr[a-z]*[\\.\\-_ ]*dt', 'le[a-z]*[\\.\\-_ ]*tr'],\n",
    "    IntSpecFitDistCuts:    ['fit[a-z]*[\\.\\-_ ]*d', 'tr[a-z]*[\\.\\-_ ]*[a-z]*[\\.\\-_ ]*mod'],\n",
    "    IntSpecDiscrDistCuts:  ['dis[a-z]*[\\.\\-_ ]*d', 'tr[a-z]*[\\.\\-_ ]*[a-z]*[\\.\\-_ ]*dis']}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert ads.DSAnalyser.userSpec2ParamNames(['key fn', 'série-aj', 'est.crit.', 'ConfInt',\n",
    "                                           'fit d', 'disc d', 'min dist', 'maxd'], int2UserSpecREs) \\\n",
    "       == [IntSpecEstimKeyFn, IntSpecEstimAdjustFn, IntSpecEstimCriterion, IntSpecCVInterval,\n",
    "           IntSpecFitDistCuts, IntSpecDiscrDistCuts, IntSpecMinDist, IntSpecMaxDist]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. _explicitParamSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "passIdCol = 'Passage'\n",
    "effortCol = 'Effort'\n",
    "\n",
    "sampleDistCol = 'distMem'\n",
    "sampleDecCols=[effortCol, sampleDistCol]\n",
    "\n",
    "sampleSelCols = ['Espèce', passIdCol, 'Adulte', 'Durée']\n",
    "sampleIndCol = 'IndSamp'\n",
    "\n",
    "varIndCol = 'IndAnlys'\n",
    "anlysAbbrevCol = 'AbrevAnlys'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Via combinaisons implicites, par fichier.\n",
    "dfExplParamSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols = \\\n",
    "    ads.DSAnalyser._explicitParamSpecs(implParamSpecs='refin/ACDC2019-Naturalist-extrait-SpecsAnalyses.xlsx',\n",
    "                                       int2UserSpecREs=int2UserSpecREs,\n",
    "                                       sampleSelCols=sampleSelCols, abbrevCol=anlysAbbrevCol,\n",
    "                                       abbrevBuilder=analysisAbbrev, anlysIndCol=varIndCol,\n",
    "                                       sampleIndCol=sampleIndCol, dropDupes=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(dfExplParamSpecs), userParamSpecCols, intParamSpecCols, unmUserParamSpecCols)\n",
    "\n",
    "assert len(dfExplParamSpecs) == 48\n",
    "assert userParamSpecCols == ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "assert intParamSpecCols == ['EstimKeyFn', 'EstimAdjustFn', 'MinDist', 'MaxDist', 'FitDistCuts']\n",
    "assert unmUserParamSpecCols == []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfExplParamSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Via combinaisons explicites, par dataframe, avec doublons nettoyés, et colonnes neutres traversantes.\n",
    "dfExplParamSpecs.drop(columns=[varIndCol, anlysAbbrevCol, sampleIndCol], inplace=True)\n",
    "dfExplParamSpecs = dfExplParamSpecs.append(dfExplParamSpecs, ignore_index=True)  # Pleins de doublons !\n",
    "dfExplParamSpecs['AvecTronc'] = dfExplParamSpecs[['TrGche', 'TrDrte']].apply(lambda s: s.isnull().all(), axis='columns')  # Neutre 1\n",
    "dfExplParamSpecs['AbrevEsp'] = dfExplParamSpecs['Espèce'].apply(lambda s: ''.join(m[:4] for m in s.split()))  # Neutre 2\n",
    "dfExplParamSpecs                                                           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes neutres non signalées, doublons conservés\n",
    "dfExplParamSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols = \\\n",
    "    ads.DSAnalyser._explicitParamSpecs(dfExplParamSpecs=dfExplParamSpecs, int2UserSpecREs=int2UserSpecREs,\n",
    "                                       sampleSelCols=sampleSelCols, abbrevCol=anlysAbbrevCol,\n",
    "                                       abbrevBuilder=analysisAbbrev, anlysIndCol=varIndCol,\n",
    "                                       sampleIndCol=sampleIndCol, dropDupes=False)\n",
    "\n",
    "print(len(dfExplParamSpecs), userParamSpecCols, intParamSpecCols, unmUserParamSpecCols)\n",
    "\n",
    "assert len(dfExplParamSpecs) == 2*48\n",
    "assert userParamSpecCols == ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "assert intParamSpecCols == ['EstimKeyFn', 'EstimAdjustFn', 'MinDist', 'MaxDist', 'FitDistCuts']\n",
    "assert unmUserParamSpecCols == ['AvecTronc', 'AbrevEsp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes neutres signalées, doublons supprimés\n",
    "dfExplParamSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols = \\\n",
    "    ads.DSAnalyser._explicitParamSpecs(dfExplParamSpecs=dfExplParamSpecs, int2UserSpecREs=int2UserSpecREs,\n",
    "                                       sampleSelCols=sampleSelCols, abbrevCol=anlysAbbrevCol,\n",
    "                                       abbrevBuilder=analysisAbbrev, anlysIndCol=varIndCol,\n",
    "                                       sampleIndCol=sampleIndCol, anlysSpecCustCols=['AvecTronc', 'AbrevEsp'],\n",
    "                                       dropDupes=True)\n",
    "\n",
    "print(len(dfExplParamSpecs), userParamSpecCols, intParamSpecCols, unmUserParamSpecCols)\n",
    "\n",
    "assert len(dfExplParamSpecs) == 48\n",
    "assert userParamSpecCols == ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "assert intParamSpecCols == ['EstimKeyFn', 'EstimAdjustFn', 'MinDist', 'MaxDist', 'FitDistCuts']\n",
    "assert unmUserParamSpecCols == []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfExplParamSpecs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10. MCDSZerothOrderTruncationOptimisation class and bases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "OptimClass = ads.MCDSZerothOrderTruncationOptimisation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Low level static and class methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One real life analysis result (no post-computed columns), grabbed from valtests\n",
    "sResults = pd.Series(\n",
    "{('header (head)', 'NumAnlys', 'Value'): 0,\n",
    " ('header (head)', 'NumEchant', 'Value'): 0,\n",
    " ('header (sample)', 'Espèce', 'Value'): 'Sylvia atricapilla',\n",
    " ('header (sample)', 'Passage', 'Value'): 'a+b',\n",
    " ('header (sample)', 'Adulte', 'Value'): 'm',\n",
    " ('header (sample)', 'Durée', 'Value'): '5mn',\n",
    " ('header (tail)', 'AbrevAnlys', 'Value'): 'SylvAtri-ab-m-5mn-hno-cos',\n",
    " ('sample stats', 'total number of observations', 'Value'): 261.0,\n",
    " ('sample stats', 'minimal observation distance', 'Value'): 10.843323181859,\n",
    " ('sample stats', 'maximal observation distance', 'Value'): 488.187599344441,\n",
    " ('parameters', 'estimator key function', 'Value'): 'HNORMAL',\n",
    " ('parameters', 'estimator adjustment series', 'Value'): 'COSINE',\n",
    " ('parameters', 'estimator selection criterion', 'Value'): 'AIC',\n",
    " ('parameters', 'CV interval', 'Value'): 95,\n",
    " ('parameters', 'left truncation distance', 'Value'): None,\n",
    " ('parameters', 'right truncation distance', 'Value'): None,\n",
    " ('parameters', 'model fitting distance cut points', 'Value'): None,\n",
    " ('run output', 'run status', 'Value'): 1,\n",
    " ('run output', 'start time', 'Value'): pd.Timestamp('2021-11-15 20:09:32.099857'),\n",
    " ('run output', 'elapsed time', 'Value'): 1.2119469999999999,\n",
    " ('run output', 'run folder', 'Value'): 'tmp/mcds-anlr/SylvAtri-ab-m-5mn-hno-cos-on2o1uv0',\n",
    " ('encounter rate', 'number of observations (n)', 'Value'): 261.0,\n",
    " ('encounter rate', 'number of samples (k)', 'Value'): 96.0,\n",
    " ('encounter rate', 'effort (L or K or T)', 'Value'): 190.0,\n",
    " ('encounter rate', 'encounter rate (n/L or n/K or n/T)', 'Value'): 1.373684,\n",
    " ('encounter rate', 'encounter rate (n/L or n/K or n/T)', 'Cv'): 0.08385102,\n",
    " ('encounter rate', 'encounter rate (n/L or n/K or n/T)', 'Lcl'): 1.163372,\n",
    " ('encounter rate', 'encounter rate (n/L or n/K or n/T)', 'Ucl'): 1.622017,\n",
    " ('encounter rate', 'encounter rate (n/L or n/K or n/T)', 'Df'): 95.0,\n",
    " ('encounter rate', 'left truncation distance', 'Value'): 0.0,\n",
    " ('encounter rate', 'right truncation distance (w)', 'Value'): 488.1876,\n",
    " ('encounter rate', 'observation rate', 'Value'): 100.0,\n",
    " ('detection probability', 'total number of parameters (m)', 'Value'): 2.0,\n",
    " ('detection probability', 'Delta AIC', 'Value'): 16.016000000000076,\n",
    " ('detection probability', 'AIC value', 'Value'): 2886.089,\n",
    " ('detection probability', 'chi-square test probability (distance set 1)', 'Value'): 3.117323e-05,\n",
    " ('detection probability', 'chi-square test probability (distance set 2)', 'Value'): 7.510185e-06,\n",
    " ('detection probability', 'chi-square test probability (distance set 3)', 'Value'): 6.556511e-07,\n",
    " ('detection probability', 'f(0) or h(0)', 'Value'): 0.0001421544,\n",
    " ('detection probability', 'f(0) or h(0)', 'Cv'): 0.04541967,\n",
    " ('detection probability', 'f(0) or h(0)', 'Lcl'): 0.0001299982,\n",
    " ('detection probability', 'f(0) or h(0)', 'Ucl'): 0.0001554473,\n",
    " ('detection probability', 'f(0) or h(0)', 'Df'): 259.0,\n",
    " ('detection probability', 'probability of detection (Pw)', 'Value'): 0.05903319,\n",
    " ('detection probability', 'probability of detection (Pw)', 'Cv'): 0.04541968,\n",
    " ('detection probability', 'probability of detection (Pw)', 'Lcl'): 0.05398504,\n",
    " ('detection probability', 'probability of detection (Pw)', 'Ucl'): 0.06455339,\n",
    " ('detection probability', 'probability of detection (Pw)', 'Df'): 259.0,\n",
    " ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Value'): 118.6137,\n",
    " ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Cv'): 0.02270984,\n",
    " ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Lcl'): 113.4269,\n",
    " ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Ucl'): 124.0377,\n",
    " ('detection probability', 'effective strip width (ESW) or effective detection radius (EDR)', 'Df'): 259.0,\n",
    " ('detection probability', 'AICc', 'Value'): 2886.136,\n",
    " ('detection probability', 'BIC', 'Value'): 2893.218,\n",
    " ('detection probability', 'Log likelihood', 'Value'): -1441.045,\n",
    " ('detection probability', 'Kolmogorov-Smirnov test probability', 'Value'): 0.002342608,\n",
    " ('detection probability', 'Cramér-von Mises (uniform weighting) test probability', 'Value'): 0.05,\n",
    " ('detection probability', 'Cramér-von Mises (cosine weighting) test probability', 'Value'): 0.05,\n",
    " ('detection probability', 'key function type', 'Value'): 'HNORMAL',\n",
    " ('detection probability', 'adjustment series type', 'Value'): 'COSINE',\n",
    " ('detection probability', 'number of key function parameters (NKP)', 'Value'): 1.0,\n",
    " ('detection probability', 'number of adjustment term parameters (NAP)', 'Value'): 1.0,\n",
    " ('detection probability', 'number of covariate parameters (NCP)', 'Value'): 0.0,\n",
    " ('density/abundance', 'density of animals', 'Value'): 0.3107902,\n",
    " ('density/abundance', 'density of animals', 'Delta Cv'): 0.0,\n",
    " ('density/abundance', 'density of animals', 'Cv'): 0.09536216,\n",
    " ('density/abundance', 'density of animals', 'Lcl'): 0.2575356,\n",
    " ('density/abundance', 'density of animals', 'Ucl'): 0.375057,\n",
    " ('density/abundance', 'density of animals', 'Df'): 154.0611})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference values for post-computed (or more accurately \"postProcessed analysis results\") columns\n",
    "sRefPostCompRes = pd.Series(\n",
    "{('detection probability', 'chi-square test probability determined', 'Value'): 6.556511e-07,\n",
    " ('combined quality', 'balanced 1', 'Value'): 0.023036527036689274,\n",
    " ('combined quality', 'balanced 2', 'Value'): 0.03589404558147751,\n",
    " ('combined quality', 'balanced 3', 'Value'): 0.03622969940156518,\n",
    " ('combined quality', 'more Chi2', 'Value'): 0.010767931727518337,\n",
    " ('combined quality', 'more KS', 'Value'): 0.026724713123070645,\n",
    " ('combined quality', 'more DCv', 'Value'): 0.0518740755014167})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Class method _postProcessAnalysisResults: Post-process analysis results\n",
    "# a. Standard case with all needed MCDS outputs presents\n",
    "sUpdRes = OptimClass._postProcessAnalysisResults(sResults)\n",
    "assert sUpdRes[sRefPostCompRes.index].eq(sRefPostCompRes).all()\n",
    "\n",
    "# b. Other cases with missing MCDS outputs : expecting small (bad) values for quality indicators\n",
    "RSClass = ads.MCDSAnalysisResultsSet\n",
    "iResPostCompQuaRes = sRefPostCompRes.index.drop(('detection probability', 'chi-square test probability determined', 'Value'))\n",
    "for missingResCol in [RSClass.CLNAdjPars, RSClass.CLNTotPars, RSClass.CLNObs, RSClass.CLNTotObs, \n",
    "                      RSClass.CLKS, RSClass.CLCvMUw, RSClass.CLCvMCw, RSClass.CLDCv]:\n",
    "    print(missingResCol)\n",
    "    sUpdRes = OptimClass._postProcessAnalysisResults(sResults.drop(missingResCol))\n",
    "    assert sUpdRes[iResPostCompQuaRes].lt(0.06).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Debug\n",
    "#missingResCol, sUpdRes[iResPostCompQuaRes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Class method _getAnalysisResultValue: compute function(results) to optimize value\n",
    "KMinimizeAnlysResults = dict(chi2=False, ks=False, cvmuw=False, cvmcw=False, dcv=True,\n",
    "                             balq1=False, balq2=False, balq3=False,\n",
    "                             balqc2=False, balqks=False, balqcv=False)\n",
    "assert set(KMinimizeAnlysResults.keys()) == set(OptimClass.AnlysResultsIndex.keys())\n",
    "\n",
    "for expr, minze in KMinimizeAnlysResults.items():\n",
    "    anlysResValue = OptimClass._getAnalysisResultValue(resultExpr=expr, sResults=sResults, invalidValue=None)\n",
    "    lblRes = OptimClass.AnlysResultsIndex[expr][0]\n",
    "    print(expr, minze, anlysResValue)\n",
    "    assert (lblRes in sRefPostCompRes.index and anlysResValue == sRefPostCompRes[lblRes]) or anlysResValue == sResults[lblRes]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Optimisations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data set to play with\n",
    "sds = ads.SampleDataSet(source=pl.Path('refin', 'ACDC2019-Papyrus-ALAARV-saisie-ttes-cols.xlsx'),\n",
    "                        decimalFields=['EFFORT', 'DISTANCE', 'NOMBRE'])\n",
    "dict(nSights=len(sds), nSightDist=len(sds.dfData['DISTANCE'].dropna()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MCDS engine\n",
    "eng = ads.MCDSEngine(workDir=tmpDir / 'mcds-zooption')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# All variant truncation params, absolute discrDistCuts (dict, tuple, list intervals) ; submit times, onlyBest, termExprValue\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=False, #autoClean=False,\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='HNO', estimAdjustFn='COS', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              minDist=(0, 50), maxDist=dict(min=150, max=200),\n",
    "                                              fitDistCutsFctr=[0.5, 1.5], discrDistCuts=dict(min=3, max=8),\n",
    "                                              expr2Optimise='chi2', minimiseExpr=False,\n",
    "                                              maxIters=30, termExprValue=0.5)\n",
    "\n",
    "zoption.submit(times=3, onlyBest=2)\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 2 and dfRes[['SetupStatus', 'SubmitStatus']].isnull().all().all() \\\n",
    "       and dfRes[['NFunEvals', 'chi2']] \\\n",
    "            .apply(lambda rs: rs['NFunEvals'] == 30 or (1 <= rs['NFunEvals'] < 30 and rs['chi2'] >= 0.5), axis='columns').all()\n",
    "# TODO: Add more auto-checks ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All variant truncation params, absolute fitDistCuts ; Interval interval\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=False, autoClean=False,\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='HNO', estimAdjustFn='COS', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              minDist=(0, 50), maxDist=ads.Interval(150, 200),\n",
    "                                              fitDistCuts=(10, 20), discrDistCutsFctr=(0.5, 1.5),\n",
    "                                              expr2Optimise='chi2', minimiseExpr=False, maxIters=6)\n",
    "\n",
    "zoption.submit()\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 1 and dfRes.loc[0, ['SetupStatus', 'SubmitStatus', 'NFunEvals']].fillna('NaN').eq(('NaN', 'NaN', 6)).all()\n",
    "# TODO: Add more auto-checks ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only maxDist variant truncation param\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=True, autoClean=False,\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='HAZ', estimAdjustFn='POLY', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              maxDist=dict(min=150, max=200),\n",
    "                                              expr2Optimise='ks', minimiseExpr=False, maxIters=5)\n",
    "\n",
    "zoption.submit()\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 1 and dfRes.loc[0, ['SetupStatus', 'SubmitStatus', 'NFunEvals']].fillna('NaN').eq(('NaN', 'NaN', 5)).all()\n",
    "# TODO: Add more auto-checks ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only minDist variant truncation param, others absent\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=False,\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='UNI', estimAdjustFn='POLY', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              minDist=(0, 50),\n",
    "                                              expr2Optimise='chi2*ks', minimiseExpr=False, maxIters=5)\n",
    "\n",
    "zoption.submit()\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 1 and dfRes.loc[0, ['SetupStatus', 'SubmitStatus', 'NFunEvals']].fillna('NaN').eq(('NaN', 'NaN', 5)).all()\n",
    "# TODO: Add more auto-checks ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Variant minDist, maxDist, fitDistCutsFctr, and const discrDistCuts\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=False,\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='HNO', estimAdjustFn='COS', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              minDist=(0, 50), maxDist=dict(min=150, max=200),\n",
    "                                              fitDistCutsFctr=[0.5, 1.5], discrDistCuts=7,\n",
    "                                              expr2Optimise='balq3', minimiseExpr=False,\n",
    "                                              maxIters=30, termExprValue=0.5)\n",
    "\n",
    "zoption.submit(times=3, onlyBest=2)  # => max 3 x 30 = 90 total analyses\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 2 and dfRes[['SetupStatus', 'SubmitStatus']].isnull().all().all() \\\n",
    "       and ((dfRes.NFunEvals.eq(90 / 2).all() and dfRes.balq3.lt(0.5).all()) \\\n",
    "            or dfRes[['NFunEvals', 'balq3']] \\\n",
    "                .apply(lambda rs: 1 <= rs.NFunEvals < 90 / 2 and rs.balq3 >= 0.5, axis='columns').any())\n",
    "# TODO: Add more auto-checks ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Variant minDist, maxDist, discrDistCutsFctr, const fitDistCuts\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=False, autoClean=False,\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='HNO', estimAdjustFn='COS', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              minDist=(0, 50), maxDist=ads.Interval(150, 200),\n",
    "                                              fitDistCuts=15, discrDistCutsFctr=(0.5, 1.5),\n",
    "                                              expr2Optimise='balq1', minimiseExpr=False, maxIters=6)\n",
    "\n",
    "zoption.submit()\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 1 and dfRes.loc[0, ['SetupStatus', 'SubmitStatus', 'NFunEvals']].fillna('NaN').eq(('NaN', 'NaN', 6)).all()\n",
    "# TODO: Add more auto-checks ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Variant maxDist, const minDist, others absent\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=True, autoClean=False,\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='HAZ', estimAdjustFn='POLY', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              maxDist=dict(min=150, max=200), minDist=20,\n",
    "                                              expr2Optimise='1-balqcv', minimiseExpr=True, maxIters=5)\n",
    "\n",
    "zoption.submit()\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 1 and dfRes.loc[0, ['SetupStatus', 'SubmitStatus', 'NFunEvals']].fillna('NaN').eq(('NaN', 'NaN', 5)).all()\n",
    "# TODO: Add more auto-checks ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Variant minDist, const maxDist, others absent\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=False,\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='UNI', estimAdjustFn='POLY', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              minDist=(0, 50), maxDist=200.0,\n",
    "                                              expr2Optimise='(chi2*ks/dcv)**(1/3.)', minimiseExpr=False, maxIters=5)\n",
    "\n",
    "zoption.submit()\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 1 and dfRes.loc[0, ['SetupStatus', 'SubmitStatus', 'NFunEvals']].fillna('NaN').eq(('NaN', 'NaN', 5)).all()\n",
    "# TODO: Add more auto-checks ?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup error, no real run\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=False,\n",
    "                                              error='Setup error !',\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='UNI', estimAdjustFn='POLY', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              minDist=(0, 50),\n",
    "                                              expr2Optimise='(balq2+balqc2)/2', minimiseExpr=False, maxIters=5)\n",
    "\n",
    "zoption.submit()\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 1 and dfRes.loc[0, ['SetupStatus', 'SubmitStatus', 'NFunEvals']].eq(('Setup error !', None, 0)).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Submit error, no real run\n",
    "zoption = \\\n",
    "    ads.MCDSZerothOrderTruncationOptimisation(engine=eng, name='alarv', logData=False,\n",
    "                                              sampleDataSet=sds, distanceField='DISTANCE',\n",
    "                                              estimKeyFn='UNI', estimAdjustFn='POLY', \n",
    "                                              estimCriterion='AIC', cvInterval=95,\n",
    "                                              minDist=(0, 50),\n",
    "                                              expr2Optimise='cvmuw*cvmcw*balqks', minimiseExpr=False, maxIters=5)\n",
    "\n",
    "zoption.submit(error='Submit error !')\n",
    "\n",
    "dfRes = zoption.getResults()\n",
    "display(dfRes)\n",
    "\n",
    "assert len(dfRes) == 1 and dfRes.loc[0, ['SetupStatus', 'SubmitStatus', 'NFunEvals']].eq((None, 'Submit error !', 0)).all()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Done."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eng.shutdown()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 11. DSParamsOptimiser abstract class \n",
    "\n",
    "(class and static methods)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "adspo = ads.DSParamsOptimiser"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. _parseUserSpec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defs for param. spec. mini-language\n",
    "auto = adspo.Auto()\n",
    "def dist(min, max):\n",
    "    return adspo.DistInterval(int(min), int(max))\n",
    "def quant(pct):\n",
    "    return adspo.OutliersMethod('quant', int(pct))\n",
    "def tucquant(pct):\n",
    "    return adspo.OutliersMethod('tucquant', float(pct))\n",
    "def mult(min, max):\n",
    "    return adspo.MultInterval(float(min), float(max))\n",
    "def abs(min, max):\n",
    "    return adspo.AbsInterval(int(min), int(max))\n",
    "def min(expr):\n",
    "    return dict(op='min', expr=expr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse spec : no error (note: look at case ;-).\n",
    "for spec in [5, 12.0, 'auto', 'Auto', 'dist(5, 12)', 'quant(8)', 'QUANT(12)', 'tucquant(5)', 'mult(1.4, 7.3)', 'Abs(4, 10)']:\n",
    "    r = adspo._parseUserSpec(spec, \n",
    "                             globals=dict(Auto=adspo.Auto,\n",
    "                                          DistInterval=adspo.DistInterval,\n",
    "                                          AbsInterval=adspo.AbsInterval,\n",
    "                                          MultInterval=adspo.MultInterval,\n",
    "                                          OutliersMethod=adspo.OutliersMethod),\n",
    "                             locals=dict(auto=auto, dist=dist, quant=quant, tucquant=tucquant,\n",
    "                                         mult=mult, abs=abs))\n",
    "    assert r[0] is None\n",
    "    print(spec, '=>', ', '.join(str(x) for x in r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse spec : errors because of bad output types.\n",
    "for spec in [1, 6.0, 'auto', 'dist(5, 12)', 'quant(8)', 'tucquant(5)', 'mult(1.4, 7.3)', 'abs(4, 10)']:\n",
    "    r = adspo._parseUserSpec(spec, \n",
    "                             globals=dict(Auto=adspo.Auto,\n",
    "                                          DistInterval=adspo.DistInterval,\n",
    "                                          AbsInterval=adspo.AbsInterval,\n",
    "                                          MultInterval=adspo.MultInterval,\n",
    "                                          OutliersMethod=adspo.OutliersMethod),\n",
    "                             locals=dict(auto=auto, dist=dist, quant=quant, tucquant=tucquant,\n",
    "                                         mult=mult, abs=abs),\n",
    "                             errIfNotA=[dict])\n",
    "    assert r[1] is None\n",
    "    print(spec, '=>', ', '.join(str(x) for x in r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse spec : empty and no error.\n",
    "for spec in [None, np.nan, '', '   ']:\n",
    "    r = adspo._parseUserSpec(spec, \n",
    "                             globals=dict(Auto=adspo.Auto,\n",
    "                                          DistInterval=adspo.DistInterval,\n",
    "                                          AbsInterval=adspo.AbsInterval,\n",
    "                                          MultInterval=adspo.MultInterval,\n",
    "                                          OutliersMethod=adspo.OutliersMethod),\n",
    "                             locals=dict(auto=auto, dist=dist, quant=quant, tucquant=tucquant,\n",
    "                                         mult=mult, abs=abs),\n",
    "                             nullOrEmpty='rien', errIfNotA=[dict]) # Note that errIfNotA is ignored (feature).\n",
    "    assert r[0] is None and r[1] == 'rien'\n",
    "    print(spec, '=>', ', '.join(str(x) for x in r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse spec : oneStrArg and no error.\n",
    "for spec in ['min(ks*chi2/12)']:\n",
    "    r = adspo._parseUserSpec(spec, \n",
    "                             globals=dict(),\n",
    "                             locals=dict(min=min),\n",
    "                             oneStrArg=True)\n",
    "    assert r[0] is None and r[1] == dict(op='min', expr='ks*chi2/12')\n",
    "    print(spec, '=>', ', '.join(str(x) for x in r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parse spec : errors.\n",
    "for spec in ['dist(5m, 12m)', 'quant(8%)', 'tucquant(t)', 'tuckey(5)', 'mult(1,4, 7.3)', 'abs(4, \\'m\\')']:\n",
    "    r = adspo._parseUserSpec(spec, \n",
    "                             globals=dict(Auto=adspo.Auto,\n",
    "                                          DistInterval=adspo.DistInterval,\n",
    "                                          AbsInterval=adspo.AbsInterval,\n",
    "                                          MultInterval=adspo.MultInterval,\n",
    "                                          OutliersMethod=adspo.OutliersMethod),\n",
    "                             locals=dict(auto=auto, dist=dist, quant=quant, tucquant=tucquant,\n",
    "                                         mult=mult, abs=abs))\n",
    "    assert r[1] is None\n",
    "    print(spec, '=>', ', '.join(str(x) for x in r))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. _parseDistTruncationUserSpec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# No error.\n",
    "r = adspo._parseDistTruncationUserSpec(2.0, errIfNotA=[float])\n",
    "print(r)\n",
    "assert r == (None, 2.0)\n",
    "             \n",
    "r = adspo._parseDistTruncationUserSpec(7, errIfNotA=[int])\n",
    "print(r)\n",
    "assert r == (None, 7)\n",
    "             \n",
    "r = adspo._parseDistTruncationUserSpec('auto', errIfNotA=[adspo.Auto])\n",
    "print(r)\n",
    "assert r == (None, adspo.Auto())\n",
    "             \n",
    "r = adspo._parseDistTruncationUserSpec('quant(5)', errIfNotA=[adspo.OutliersMethod])\n",
    "print(r)\n",
    "assert r == (None, adspo.OutliersMethod('quant', 5))\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('abs(8, 12)', errIfNotA=[adspo.AbsInterval])\n",
    "print(r)\n",
    "assert r == (None, adspo.AbsInterval(8, 12))\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('dist(0, 70)', errIfNotA=[adspo.DistInterval])\n",
    "print(r)\n",
    "assert r == (None, adspo.DistInterval(0, 70))\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('mult(0.6, 1.2)', errIfNotA=[adspo.MultInterval])\n",
    "print(r)\n",
    "assert r == (None, adspo.MultInterval(0.6, 1.2))\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('tucquant(2.5)')\n",
    "print(r)\n",
    "assert r == (None, adspo.OutliersMethod('tucquant', 2.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bad type errors.\n",
    "r = adspo._parseDistTruncationUserSpec('auto', errIfNotA=(adspo.AbsInterval, adspo.MultInterval))\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('quant(5)', errIfNotA=[adspo.Auto])\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('abs(8, 12)', errIfNotA=(adspo.OutliersMethod,))\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('mult(0.6, 1.2)', errIfNotA=(adspo.DistInterval, adspo.OutliersMethod))\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('tucquant(2.5)', errIfNotA=(adspo.DistInterval, adspo.MultInterval))\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parsing errors.\n",
    "r = adspo._parseDistTruncationUserSpec('autox')\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('tuckey(5)')\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('abs(12)')\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('mult(0.6, x)')\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None\n",
    "\n",
    "r = adspo._parseDistTruncationUserSpec('tucquant(2.5%)')\n",
    "print(r[0])\n",
    "assert r[0] is not None and r[1] is None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 12. MCDSTruncationOptimiser abstract class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "adsto = ads.MCDSTruncationOptimiser"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Individualised data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countCols =  ['nMalAd10', 'nAutAd10', 'nMalAd5', 'nAutAd5']\n",
    "\n",
    "def count2AdultCat(sCounts):\n",
    "    return 'm' if 'Mal' in sCounts[sCounts > 0].index[0] else 'a'\n",
    "\n",
    "def count2DurationCat(sCounts):\n",
    "    return '5mn' if '5' in sCounts[sCounts > 0].index[0] else '10mn'\n",
    "\n",
    "fds = ads.FieldDataSet(source='refin/ACDC2019-Naturalist-extrait-ObsBrutesAvecDist.txt',\n",
    "                       importDecFields=['distMem'], countCols=countCols,\n",
    "                       addMonoCatCols={ 'Adulte': count2AdultCat, 'Durée': count2DurationCat })\n",
    "\n",
    "dfObsIndiv = fds.individualise()\n",
    "\n",
    "dfObsIndiv.drop(columns=countCols, inplace=True)\n",
    "\n",
    "dfObsIndiv.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transectPlaceCols = ['Point']\n",
    "passIdCol = 'Passage'\n",
    "effortCol = 'Effort'\n",
    "\n",
    "sampleDistCol = 'distMem'\n",
    "sampleCols = [passIdCol, 'Adulte', 'Durée']\n",
    "sampleDecCols=[effortCol, sampleDistCol]\n",
    "\n",
    "sampleSelCols = ['Espèce'] + sampleCols\n",
    "\n",
    "varIndCol = 'IndAnlys'\n",
    "anlysAbbrevCol = 'AbrevAnlys'\n",
    "\n",
    "dSurveyArea = dict(Zone='ACDC', Surface='2400')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show samples\n",
    "dfObsIndiv[sampleCols].drop_duplicates()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Ctor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check run method and time-out support\n",
    "try:\n",
    "    optr = ads.MCDSTruncationOptimiser \\\n",
    "                    (dfObsIndiv, effortConstVal=1, dSurveyArea=dSurveyArea, \n",
    "                     transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                     sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                     workDir=tmpDir / 'mcds-optr', runMethod='os.system', runTimeOut=120)\n",
    "except AssertionError as exc:\n",
    "    if re.search(\"Can't care about .+s execution time limit\", str(exc)):\n",
    "        print('Good: Expected refuse to work for incompatible params')\n",
    "    else:\n",
    "        raise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#An operational one for checks below\n",
    "optr = ads.MCDSTruncationOptimiser \\\n",
    "                (dfObsIndiv, effortConstVal=1, dSurveyArea=dSurveyArea, \n",
    "                 transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                 sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                 distanceUnit='Meter', areaUnit='Hectare',\n",
    "                 surveyType='Point', distanceType='Radial', clustering=False,\n",
    "                 resultsHeadCols=dict(before=[varIndCol], sample=sampleSelCols, after=[anlysAbbrevCol]),\n",
    "                 abbrevCol=anlysAbbrevCol, workDir=tmpDir / 'mcds-optr', logData=False,                 \n",
    "                 defEstimKeyFn='HNO', defEstimAdjustFn='COS',\n",
    "                 defEstimCriterion='AIC', defCVInterval=95,\n",
    "                 defExpr2Optimise='chi2', defMinimiseExpr=False,\n",
    "                 defOutliersMethod='tucquant', defOutliersQuantCutPct=5,\n",
    "                 defFitDistCutsFctr=dict(min=2/3, max=3/2),\n",
    "                 defDiscrDistCutsFctr=dict(min=1/3, max=1),\n",
    "                 defSubmitTimes=4, defSubmitOnlyBest=2,\n",
    "                 dDefOptimCoreParams=dict(core='zoopt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. getAnalysisOptimExprParams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Spec is present\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecExpr2Optimise: 'min(ks*chi2/12)' })\n",
    "r = optr.getAnalysisOptimExprParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(minimiseExpr=True, expr2Optimise='ks*chi2/12')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Spec is null\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecExpr2Optimise: None })\n",
    "r = optr.getAnalysisOptimExprParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(minimiseExpr=False, expr2Optimise='chi2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Spec is absent\n",
    "sAnIntSpec = pd.Series(dtype=float)\n",
    "r = optr.getAnalysisOptimExprParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(minimiseExpr=False, expr2Optimise='chi2')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. getAnalysisFixedParams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All specs present\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecEstimKeyFn:'HNO', adsto.IntSpecEstimAdjustFn:'POLY',\n",
    "                         adsto.IntSpecEstimCriterion:'AIC', adsto.IntSpecCVInterval:97 })\n",
    "r = optr.getAnalysisFixedParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(estimKeyFn='HNO', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=97)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some specs absent => default values\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecEstimKeyFn:'UNI', adsto.IntSpecEstimAdjustFn:'POLY'})\n",
    "r = optr.getAnalysisFixedParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(estimKeyFn='UNI', estimAdjustFn='POLY', estimCriterion='AIC', cvInterval=95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e. getAnalysisOptimedParams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get a \"random\" sample from indiv. data set + compute some base figures for checking results\n",
    "sAnSpec = pd.Series({'Espèce': 'Alauda arvensis', 'Passage': 'a+b', 'Adulte': 'm+a', 'Durée': '10mn'})\n",
    "sds = optr._mcDataSet.sampleDataSet(sAnSpec[sampleCols])\n",
    "sSampleDistances = sds.dfData[sampleDistCol].dropna()\n",
    "\n",
    "sqd = np.sqrt(len(sSampleDistances.dropna()))\n",
    "dMin = sSampleDistances.min()\n",
    "dMax = sSampleDistances.max()\n",
    "\n",
    "print(f'* Sample distances: dMin={dMin}, dMax={dMax}, ...')\n",
    "print(f'  {len(sSampleDistances)} sample distances =>\\n{sorted(sSampleDistances.values)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All present and variant (check computations) 1\n",
    "# a. Call method\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecMinDist:'auto', adsto.IntSpecMaxDist:'quant(5)',\n",
    "                         adsto.IntSpecFitDistCuts:'abs(8, 12)', adsto.IntSpecDiscrDistCuts:'mult(0.6, 1.2)',\n",
    "                         adsto.IntSpecOutliersMethod:'tucquant(2.5)'})\n",
    "e, r = optr.getAnalysisOptimedParams(sAnIntSpec, sSampleDistances)\n",
    "\n",
    "assert e is None\n",
    "\n",
    "sr = str({ k:str(v) for k,v in r.items() })\n",
    "print('Actual result   :', sr)\n",
    "\n",
    "# b. Compute theorical result\n",
    "qLeft, qRight = np.percentile(a=sSampleDistances, q=[2.5, 95])\n",
    "\n",
    "print('Base variables  :', dict(sqd=sqd, dMin=dMin, dMax=dMax, qLeft=qLeft, qRight=qRight))\n",
    "\n",
    "sol = dict(minDist=ads.Interval(dMin, qLeft), maxDist=ads.Interval(qRight, dMax),\n",
    "           fitDistCuts=ads.Interval(8, 12), discrDistCuts=ads.Interval(int(round(sqd*0.6)), int(round(sqd*1.2))))\n",
    "\n",
    "ssol = str({ k:str(v) for k,v in sol.items() })\n",
    "print('Theorical result:', ssol)\n",
    "\n",
    "# c. Check \"equality\" (for some reason, must use str repr for comparison ...)\n",
    "assert sr == ssol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliersQuantCutPct = 2.5\n",
    "d25, d75, d95, dPct = np.percentile(a=sSampleDistances, q=[25, 75, 95, 100-outliersQuantCutPct])\n",
    "minMaxDist = min(max(d95, d75 + 1.5*(d75 - d25)), dPct)\n",
    "\n",
    "dict(d25=d25, d75=d75, dSup=d75 + 1.5*(d75 - d25), d95=d95, dPct=dPct, minMaxDist=minMaxDist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get a new \"random\" sample from indiv. data set + compute some base figures for checking results\n",
    "sAnSpec = pd.Series({'Espèce': 'Alauda arvensis', 'Passage': 'a+b', 'Adulte': 'm+a', 'Durée': '5mn'})\n",
    "sds = optr._mcDataSet.sampleDataSet(sAnSpec[sampleCols])\n",
    "sSampleDistances = sds.dfData[sampleDistCol].dropna()\n",
    "\n",
    "sqd = np.sqrt(len(sSampleDistances.dropna()))\n",
    "dMin = sSampleDistances.min()\n",
    "dMax = sSampleDistances.max()\n",
    "\n",
    "print(f'* Sample distances: dMin={dMin}, dMax={dMax}, ...')\n",
    "print(f'  {len(sSampleDistances)} sample distances =>\\n{sorted(sSampleDistances.values)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All present and variant (check computations) 2\n",
    "# a. Call method\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecMinDist:'quant(5)', adsto.IntSpecMaxDist:'auto',\n",
    "                         adsto.IntSpecFitDistCuts:'mult(3/4, 5/4)', adsto.IntSpecDiscrDistCuts:'abs(4, 6)',\n",
    "                         adsto.IntSpecOutliersMethod:'tucquant(1)'})\n",
    "e, r = optr.getAnalysisOptimedParams(sAnIntSpec, sSampleDistances)\n",
    "\n",
    "assert e is None\n",
    "\n",
    "sr = str({ k:str(v) for k,v in r.items() })\n",
    "print('Actual result   :', sr)\n",
    "\n",
    "# b. Compute theorical result\n",
    "qLeft, q25, q75, q95, qRight = np.percentile(a=sSampleDistances, q=[5, 25, 75, 95, 99])\n",
    "qSup = q75 + 1.5 * (q75 - q25)\n",
    "print('Base variables  :', dict(sqd=sqd, dMin=dMin, dMax=dMax, qLeft=qLeft,\n",
    "                                q75=q75, qSup=qSup, q95=q95, qRight=qRight))\n",
    "\n",
    "sol = dict(minDist=ads.Interval(dMin, qLeft), maxDist=ads.Interval(qSup, dMax),\n",
    "           fitDistCuts=ads.Interval(int(round(sqd*3/4)), int(round(sqd*5/4))), discrDistCuts=ads.Interval(4, 6))\n",
    "\n",
    "ssol = str({ k:str(v) for k,v in sol.items() })\n",
    "print('Theorical result:', ssol)\n",
    "\n",
    "# c. Check \"equality\" (for some reason, must use str repr for comparison ...)\n",
    "assert sr == ssol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliersQuantCutPct = 1\n",
    "d25, d75, d95, dPct = np.percentile(a=sSampleDistances, q=[25, 75, 95, 100-outliersQuantCutPct])\n",
    "minMaxDist = min(max(d95, d75 + 1.5*(d75 - d25)), dPct)\n",
    "\n",
    "dict(d25=d25, d75=d75, dSup=d75 + 1.5*(d75 - d25), d95=d95, dPct=dPct, minMaxDist=minMaxDist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get a new \"random\" sample from indiv. data set + compute some base figures for checking results\n",
    "sAnSpec = pd.Series({'Espèce': 'Alauda arvensis', 'Passage': 'a+b', 'Adulte': 'm+a', 'Durée': '10mn'})\n",
    "sds = optr._mcDataSet.sampleDataSet(sAnSpec[sampleCols])\n",
    "sSampleDistances = sds.dfData[sampleDistCol].dropna()\n",
    "\n",
    "sqd = np.sqrt(len(sSampleDistances.dropna()))\n",
    "dMin = sSampleDistances.min()\n",
    "dMax = sSampleDistances.max()\n",
    "\n",
    "print(f'* Sample distances: dMin={dMin}, dMax={dMax}, ...')\n",
    "print(f'  {len(sSampleDistances)} sample distances =>\\n{sorted(sSampleDistances.values)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# All present and variant (check computations) 3\n",
    "# a. Call method\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecMinDist:'auto', adsto.IntSpecMaxDist:'auto',\n",
    "                         adsto.IntSpecFitDistCuts:'auto', adsto.IntSpecDiscrDistCuts:'auto',\n",
    "                         adsto.IntSpecOutliersMethod:'tucquant(2)'})\n",
    "e, r = optr.getAnalysisOptimedParams(sAnIntSpec, sSampleDistances)\n",
    "\n",
    "assert e is None\n",
    "\n",
    "sr = str({ k:str(v) for k,v in r.items() })\n",
    "print('Actual result   :', sr)\n",
    "\n",
    "# b. Compute theorical result\n",
    "qLeft, q95, qRight = np.percentile(a=sSampleDistances, q=[2, 95, 98])\n",
    "\n",
    "print('Base variables  :', dict(sqd=sqd, dMin=dMin, dMax=dMax, qLeft=qLeft, q95=q95, qRight=qRight))\n",
    "\n",
    "sol = dict(minDist=ads.Interval(dMin, qLeft), maxDist=ads.Interval(q95, dMax),\n",
    "           fitDistCuts=ads.Interval(int(round(sqd*2/3)), int(round(sqd*3/2))),\n",
    "           discrDistCuts=ads.Interval(int(round(sqd/3)), int(round(sqd))))\n",
    "\n",
    "ssol = str({ k:str(v) for k,v in sol.items() })\n",
    "print('Theorical result:', ssol)\n",
    "\n",
    "# c. Check \"equality\" (for some reason, must use str repr for comparison ...)\n",
    "assert sr == ssol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliersQuantCutPct = 6\n",
    "d25, d75, d95, dPct = np.percentile(a=sSampleDistances, q=[25, 75, 95, 100-outliersQuantCutPct])\n",
    "minMaxDist = min(max(d95, d75 + 1.5*(d75 - d25)), dPct)\n",
    "\n",
    "dict(d25=d25, d75=d75, dSup=d75 + 1.5*(d75 - d25), d95=d95, dPct=dPct, minMaxDist=minMaxDist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All present and variant (check computations) 4\n",
    "# a. Call method\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecMinDist:'auto', adsto.IntSpecMaxDist:'auto',\n",
    "                         adsto.IntSpecFitDistCuts:'auto', adsto.IntSpecDiscrDistCuts:'auto',\n",
    "                         adsto.IntSpecOutliersMethod:'auto'})\n",
    "e, r = optr.getAnalysisOptimedParams(sAnIntSpec, sSampleDistances)\n",
    "\n",
    "assert e is None\n",
    "\n",
    "sr = str({ k:str(v) for k,v in r.items() })\n",
    "print('Actual result   :', sr)\n",
    "\n",
    "# b. Compute theorical result\n",
    "qLeft, qRight = np.percentile(a=sSampleDistances, q=[5, 95])\n",
    "\n",
    "print('Base variables  :', dict(sqd=sqd, dMin=dMin, dMax=dMax, qLeft=qLeft, qRight=qRight))\n",
    "\n",
    "sol = dict(minDist=ads.Interval(dMin, qLeft), maxDist=ads.Interval(qRight, dMax),\n",
    "           fitDistCuts=ads.Interval(int(round(sqd*2/3)), int(round(sqd*3/2))),\n",
    "           discrDistCuts=ads.Interval(int(round(sqd/3)), int(round(sqd))))\n",
    "\n",
    "ssol = str({ k:str(v) for k,v in sol.items() })\n",
    "print('Theorical result:', ssol)\n",
    "\n",
    "# c. Check \"equality\" (for some reason, must use str repr for comparison ...)\n",
    "assert sr == ssol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All present, some variant, some consts (check computations) 1\n",
    "# a. Call method\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecMinDist:12, adsto.IntSpecMaxDist:'quant(5)',\n",
    "                         adsto.IntSpecFitDistCuts:'abs(8, 12)', adsto.IntSpecDiscrDistCuts:'mult(0.6, 1.2)',\n",
    "                         adsto.IntSpecOutliersMethod:'tucquant(2.5)'})\n",
    "e, r = optr.getAnalysisOptimedParams(sAnIntSpec, sSampleDistances)\n",
    "\n",
    "assert e is None\n",
    "\n",
    "sr = str({ k:str(v) for k,v in r.items() })\n",
    "print('Actual result   :', sr)\n",
    "\n",
    "# b. Compute theorical result\n",
    "qLeft, qRight = np.percentile(a=sSampleDistances, q=[2.5, 95])\n",
    "\n",
    "print('Base variables  :', dict(sqd=sqd, dMin=dMin, dMax=dMax, qLeft=qLeft, qRight=qRight))\n",
    "\n",
    "sol = dict(minDist=12, maxDist=ads.Interval(qRight, dMax),\n",
    "           fitDistCuts=ads.Interval(8, 12), discrDistCuts=ads.Interval(int(round(sqd*0.6)), int(round(sqd*1.2))))\n",
    "\n",
    "ssol = str({ k:str(v) for k,v in sol.items() })\n",
    "print('Theorical result:', ssol)\n",
    "\n",
    "# c. Check \"equality\" (for some reason, must use str repr for comparison ...)\n",
    "assert sr == ssol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All present, some variant, some consts (check computations) 2\n",
    "# a. Call method\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecMinDist:'quant(5)', adsto.IntSpecMaxDist:250.0,\n",
    "                         adsto.IntSpecFitDistCuts:'mult(3/4, 5/4)', adsto.IntSpecDiscrDistCuts:'abs(4, 6)',\n",
    "                         adsto.IntSpecOutliersMethod:'tucquant(1)'})\n",
    "e, r = optr.getAnalysisOptimedParams(sAnIntSpec, sSampleDistances)\n",
    "\n",
    "assert e is None\n",
    "\n",
    "sr = str({ k:str(v) for k,v in r.items() })\n",
    "print('Actual result   :', sr)\n",
    "\n",
    "# b. Compute theorical result\n",
    "qLeft, qRight = np.percentile(a=sSampleDistances, q=[5, 99])\n",
    "\n",
    "print('Base variables  :', dict(sqd=sqd, dMin=dMin, dMax=dMax, qLeft=qLeft, qRight=qRight))\n",
    "\n",
    "sol = dict(minDist=ads.Interval(dMin, qLeft), maxDist=250.0,\n",
    "           fitDistCuts=ads.Interval(int(round(sqd*3/4)), int(round(sqd*5/4))), discrDistCuts=ads.Interval(4, 6))\n",
    "\n",
    "ssol = str({ k:str(v) for k,v in sol.items() })\n",
    "print('Theorical result:', ssol)\n",
    "\n",
    "# c. Check \"equality\" (for some reason, must use str repr for comparison ...)\n",
    "assert sr == ssol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# All present, some variant, some consts (check computations) 3\n",
    "# a. Call method\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecMinDist:'auto', adsto.IntSpecMaxDist:'auto',\n",
    "                         adsto.IntSpecFitDistCuts:17, adsto.IntSpecDiscrDistCuts:'auto',\n",
    "                         adsto.IntSpecOutliersMethod:'tucquant(6)'})\n",
    "e, r = optr.getAnalysisOptimedParams(sAnIntSpec, sSampleDistances)\n",
    "\n",
    "assert e is None\n",
    "\n",
    "sr = str({ k:str(v) for k,v in r.items() })\n",
    "print('Actual result   :', sr)\n",
    "\n",
    "# b. Compute theorical result\n",
    "qLeft, qRight = np.percentile(a=sSampleDistances, q=[6, 94])\n",
    "\n",
    "print('Base variables  :', dict(sqd=sqd, dMin=dMin, dMax=dMax, qLeft=qLeft, qRight=qRight))\n",
    "\n",
    "sol = dict(minDist=ads.Interval(dMin, qLeft), maxDist=ads.Interval(qRight, dMax),\n",
    "           fitDistCuts=17, discrDistCuts=ads.Interval(int(round(sqd/3)), int(round(sqd))))\n",
    "\n",
    "ssol = str({ k:str(v) for k,v in sol.items() })\n",
    "print('Theorical result:', ssol)\n",
    "\n",
    "# c. Check \"equality\" (for some reason, must use str repr for comparison ...)\n",
    "assert sr == ssol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliersQuantCutPct = 6\n",
    "d25, d75, d95, dPct = np.percentile(a=sSampleDistances, q=[25, 75, 95, 100-outliersQuantCutPct])\n",
    "minMaxDist = min(max(d95, d75 + 1.5*(d75 - d25)), dPct)\n",
    "\n",
    "dict(d25=d25, d75=d75, dSup=d75 + 1.5*(d75 - d25), d95=d95, dPct=dPct, minMaxDist=minMaxDist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All present, some variant, some consts (check computations) 4\n",
    "# a. Call method\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecMinDist:'auto', adsto.IntSpecMaxDist:'auto',\n",
    "                         adsto.IntSpecFitDistCuts:'auto', adsto.IntSpecDiscrDistCuts:6,\n",
    "                         adsto.IntSpecOutliersMethod:'auto'})\n",
    "e, r = optr.getAnalysisOptimedParams(sAnIntSpec, sSampleDistances)\n",
    "\n",
    "assert e is None\n",
    "\n",
    "sr = str({ k:str(v) for k,v in r.items() })\n",
    "print('Actual result   :', sr)\n",
    "\n",
    "# b. Compute theorical result\n",
    "qLeft, qRight = np.percentile(a=sSampleDistances, q=[5, 95])\n",
    "\n",
    "print('Base variables  :', dict(sqd=sqd, dMin=dMin, dMax=dMax, qLeft=qLeft, qRight=qRight))\n",
    "\n",
    "sol = dict(minDist=ads.Interval(dMin, qLeft), maxDist=ads.Interval(qRight, dMax),\n",
    "           fitDistCuts=ads.Interval(int(round(sqd*2/3)), int(round(sqd*3/2))), discrDistCuts=6)\n",
    "\n",
    "ssol = str({ k:str(v) for k,v in sol.items() })\n",
    "print('Theorical result:', ssol)\n",
    "\n",
    "# c. Check \"equality\" (for some reason, must use str repr for comparison ...)\n",
    "assert sr == ssol"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### f. getOptimisationCoreParams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs not present => default from ctor\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecOptimisationCore: np.nan })\n",
    "r = optr.getOptimisationCoreParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(core='zoopt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs null => default from ctor\n",
    "sAnIntSpec = pd.Series(dtype=float)\n",
    "r = optr.getOptimisationCoreParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(core='zoopt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some specs present, with all default values ; string as last param.\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecOptimisationCore: 'zoopt(mxi=0,a=racos)'})\n",
    "r = optr.getOptimisationCoreParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(core='zoopt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some specs present, some with default values, some not, 1 non keyword param.\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecOptimisationCore: 'zoopt(80, a=racos)'})\n",
    "r = optr.getOptimisationCoreParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(core='zoopt', maxIters=80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All specs present, no default value\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecOptimisationCore: 'zoopt(a=sracos,mxi=450,tv=1,mxr=5)'})\n",
    "r = optr.getOptimisationCoreParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(core='zoopt', algorithm='sracos', maxIters=450, termExprValue=1, maxRetries=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### g. getOptimisationSubmitParams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs not present => default from ctor\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecSubmitParams: np.nan })\n",
    "r = optr.getOptimisationSubmitParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(times=4, onlyBest=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specs null => default from ctor\n",
    "sAnIntSpec = pd.Series(dtype=float)\n",
    "r = optr.getOptimisationSubmitParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(times=4, onlyBest=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some specs present, with default values\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecSubmitParams: 'times(n=9)'})\n",
    "r = optr.getOptimisationSubmitParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(times=9, onlyBest=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# All specs present, no default value\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecSubmitParams: 'times(100, b=22)'})\n",
    "r = optr.getOptimisationSubmitParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[0] is None and r[1] == dict(times=100, onlyBest=22)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bad times times\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecSubmitParams: 'times(n=0, b=22)'})\n",
    "r = optr.getOptimisationSubmitParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[1] is None and str(r[0]).find('Run times must be > 0') >= 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bad best kept values number\n",
    "sAnIntSpec = pd.Series({ adsto.IntSpecSubmitParams: 'times(2, b=0)'})\n",
    "r = optr.getOptimisationSubmitParams(sAnIntSpec)\n",
    "print(*r)\n",
    "assert r[1] is None and str(r[0]).find('Number of best kept values must be > 0') >= 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 13. MCDSAnalysisResultsSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = ads.logger('ads.anr', level=ads.INFO5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RS = ads.MCDSAnalysisResultsSet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. _groupingIntervals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "__builtin__.min, min"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Old code that was slightly modified ... for non-regression tests.\n",
    "try:\n",
    "    del min  # If somewhere the __builtin__.min function is overriden ... restore it !\n",
    "except:\n",
    "    pass\n",
    "\n",
    "def intervalsOld(sDists, minIntrvDist, maxIntrvLen, intrvEpsilon):\n",
    "\n",
    "    # For some reason, need for enforcing float dtype ... otherwise dtype='O' !?\n",
    "    sSelDist = sDists.dropna().astype(float).sort_values()\n",
    "\n",
    "    # List non-null differences between consecutive sorted distances\n",
    "    dfIntrv = pd.DataFrame(dict(dist=sSelDist.values))\n",
    "    if not dfIntrv.empty:\n",
    "\n",
    "        dfIntrv['deltaDist'] = dfIntrv.dist.diff()\n",
    "        dfIntrv.loc[dfIntrv.dist.idxmin(), 'deltaDist'] = np.inf\n",
    "        dfIntrv.dropna(inplace=True)\n",
    "        dfIntrv = dfIntrv[dfIntrv.deltaDist > 0].copy()\n",
    "\n",
    "        # Deduce start (min) and end (sup) for each such interval (left-closed, right-open)\n",
    "        dfIntrv['dMin'] = dfIntrv.loc[dfIntrv.deltaDist > minIntrvDist, 'dist']\n",
    "        dfIntrv['dSup'] = dfIntrv.loc[dfIntrv.deltaDist > minIntrvDist, 'dist'].shift(-1).dropna()\n",
    "        dfIntrv.loc[dfIntrv['dMin'].idxmax(), 'dSup'] = np.inf\n",
    "        dfIntrv.dropna(inplace=True)\n",
    "\n",
    "        dfIntrv['dSup'] = \\\n",
    "            dfIntrv['dSup'].apply(lambda supV: sSelDist[sSelDist < supV].max() + intrvEpsilon)\n",
    "\n",
    "        dfIntrv = dfIntrv[['dMin', 'dSup']].reset_index(drop=True)\n",
    "\n",
    "        # If these intervals are two wide, cut them up in equal sub-intervals and make them new intervals\n",
    "        lsNewIntrvs = list()\n",
    "        for _, sIntrv in dfIntrv.iterrows():\n",
    "\n",
    "            if sIntrv.dSup - sIntrv.dMin > maxIntrvLen:\n",
    "                #print(sIntrv.dSup, '-', sIntrv.dMin, '>', maxIntrvLen)\n",
    "                nSubIntrvs = (sIntrv.dSup - sIntrv.dMin) / maxIntrvLen\n",
    "                nSubIntrvs = int(nSubIntrvs) if nSubIntrvs - int(nSubIntrvs) < 0.5 else int(nSubIntrvs) + 1\n",
    "                subIntrvLen = (sIntrv.dSup - sIntrv.dMin) / nSubIntrvs\n",
    "                lsNewIntrvs += [pd.Series(dict(dMin=sIntrv.dMin + nInd * subIntrvLen,\n",
    "                                               dSup=min(sIntrv.dMin + (nInd + 1) * subIntrvLen,\n",
    "                                                        sIntrv.dSup)))\n",
    "                                for nInd in range(nSubIntrvs)]\n",
    "            else:\n",
    "                lsNewIntrvs.append(sIntrv)\n",
    "\n",
    "        dfIntrv = pd.DataFrame(lsNewIntrvs).reset_index(drop=True)\n",
    "        dfIntrv.sort_values(by='dMin', inplace=True)\n",
    "        dfIntrv.rename(columns=dict(dMin='vmin', dSup='vsup'), inplace=True)\n",
    "\n",
    "    return dfIntrv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "minDist = 5\n",
    "maxLen = 10\n",
    "eps = 1e-6\n",
    "\n",
    "v0 = 1\n",
    "KCases = [dict(values=[np.nan],  # empty series (after cleanup)\n",
    "               intervals=[]),\n",
    "          dict(values=[v0],  # 1 isolated value in 1 shortest interval\n",
    "               intervals=[{'vmin': v0,  'vsup': v0+eps}]),\n",
    "          dict(values=[v0, 2, 3, 8, 9, v0+maxLen, v0+maxLen*1.5-0.01, # Don't cut 1st interval\n",
    "                       22, 30, 35, np.nan, 44.9],  # 22 isolated inside 1 shortest interval\n",
    "               intervals=[{'vmin': v0,  'vsup': v0+maxLen*1.5-0.01},\n",
    "                          {'vmin': 22, 'vsup': 22+eps},\n",
    "                          {'vmin': 30, 'vsup': 35+eps},\n",
    "                          {'vmin': 44.9, 'vsup': 44.9+eps}]),\n",
    "          dict(values=[v0, 2, np.nan, 3, 8, 9, v0+maxLen, v0+maxLen*1.5, # Cut 1st interval\n",
    "                       22, 27, 35, 37, 39, 44.9],\n",
    "               intervals=[{'vmin': v0,  'vsup': 8.5},\n",
    "                          {'vmin': 8.5+eps, 'vsup': 16.0+eps},\n",
    "                          {'vmin': 22, 'vsup': 27+eps},\n",
    "                          {'vmin': 35, 'vsup': 39+eps},\n",
    "                          {'vmin': 44.9, 'vsup': 44.9+eps}]),\n",
    "         ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(f'{minDist=}, {maxLen=}, {eps=}:')\n",
    "\n",
    "for case in KCases:\n",
    "\n",
    "    print('*', case['values'], ':', end=' ')\n",
    "    dfIntrvs = RS._groupingIntervals(pd.Series(case['values']), minDist=minDist, maxLen=maxLen, epsilon=eps)\n",
    "    print(dfIntrvs.to_dict('index'))\n",
    "    dfDiff = ads.DataSet.compareDataFrames(dfIntrvs.reset_index(), pd.DataFrame(case['intervals']).reset_index(),\n",
    "                                           indexCols=['index'], dropCloser=4)\n",
    "    assert dfDiff.empty, 'Oh, oh ... not what we expected ; diff to ref= ' + str(dfDiff.to_dict('index')) \\\n",
    "                         + '\\nref= ' + str(case['intervals'])\n",
    "    \n",
    "    # Non regression, comparing to old code results\n",
    "    dfIntrvsOld = intervalsOld(pd.Series(case['values']), minIntrvDist=minDist, maxIntrvLen=maxLen, intrvEpsilon=eps)\n",
    "    dfDiffOld = ads.DataSet.compareDataFrames(dfIntrvs.reset_index(), dfIntrvsOld.reset_index(),\n",
    "                                              indexCols=['index'], dropCloser=6)\n",
    "    assert dfDiff.empty, 'Oh, oh ... not what we expected ; diff to old= ' + str(dfDiffOld.to_dict('index'))\n",
    "    \n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. _intervalIndex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RS = ads.MCDSAnalysisResultsSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eps = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfIntrvs = pd.DataFrame([{'vmin': 1, 'vsup': 8.5+eps},\n",
    "                         {'vmin': 8.5+eps, 'vsup': 16+eps},\n",
    "                         {'vmin': 22, 'vsup': 27+eps},\n",
    "                         {'vmin': 35, 'vsup': 39+eps},\n",
    "                         {'vmin': 44.9, 'vsup': 44.9+eps}])\n",
    "dfIntrvs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sValues = pd.Series([8.5, 39, 44.9, 1, 11, 16+eps, 2, np.nan, 3, 8, 8.5+eps, 9, 16, 22-eps, 27, 27+2*eps, 35, 22, 37])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute and show results\n",
    "sGroups = sValues.apply(RS._intervalIndex, dfIntervals=dfIntrvs)\n",
    "\n",
    "pd.DataFrame(dict(values=sValues, group=sGroups))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check\n",
    "assert sGroups.eq(pd.Series({0: 1, 1: 4, 2: 5, 3: 1, 4: 2, 5: -1, 6: 1, 7: 0, 8: 1, 9: 1, 10: 2,\n",
    "                             11: 2, 12: 2, 13: -1, 14: 3, 15: -1, 16: 4, 17: 3, 18: 4})).all()\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Prepare stuff for creating MCDSAnalysisResultsSet objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source / Results data\n",
    "transectPlaceCols = ['Point']\n",
    "passIdCol = 'Passage'\n",
    "effortCol = 'Effort'\n",
    "\n",
    "sampleDistCol = 'Distance'\n",
    "sampleDecCols = [effortCol, sampleDistCol]\n",
    "\n",
    "sampleNumCol = 'NumEchant'\n",
    "sampleSelCols = ['Espèce', passIdCol, 'Adulte', 'Durée']\n",
    "\n",
    "sampleAbbrevCol = 'AbrevEchant'\n",
    "\n",
    "dSurveyArea = dict(Zone='ACDC', Surface='2400')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# General DS analysis parameters\n",
    "varIndCol = 'NumAnlys'\n",
    "anlysAbbrevCol = 'AbrevAnlys'\n",
    "anlysParamCols = ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "\n",
    "distanceUnit = 'Meter'\n",
    "areaUnit = 'Hectare'\n",
    "surveyType = 'Point'\n",
    "distanceType = 'Radial'\n",
    "clustering = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results post-computation parameters\n",
    "ldTruncIntrvSpecs = [dict(col='left', minDist=5.0, maxLen=5.0), dict(col='right', minDist=25.0, maxLen=25.0)]\n",
    "truncIntrvEpsilon = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load individualised observations and actual transects\n",
    "indivObsFile = 'refin/ACDC2019-Naturalist-extrait-ObsIndiv.ods'\n",
    "\n",
    "dfObsIndiv = ads.DataSet(indivObsFile, sheet='DonnéesIndiv').dfData\n",
    "\n",
    "dfTransects = ads.DataSet(indivObsFile, sheet='Inventaires').dfData\n",
    "\n",
    "dict(indivObs=len(dfObsIndiv), transects=len(dfTransects))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# What's better to create an MCDSAnalysisResultsSet objets than a MCDSAnalyser instance ?\n",
    "anlr = \\\n",
    "    ads.MCDSAnalyser(dfObsIndiv, dfTransects=dfTransects, dSurveyArea=dSurveyArea, \n",
    "                     transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                     sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                     abbrevCol=anlysAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                     anlysIndCol=varIndCol, sampleIndCol=sampleNumCol,\n",
    "                     distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                     surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                     resultsHeadCols=dict(before=[varIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                          after=anlysParamCols + [anlysAbbrevCol]),\n",
    "                     ldTruncIntrvSpecs=ldTruncIntrvSpecs, truncIntrvEpsilon=truncIntrvEpsilon)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. _postComputeChi2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# i. Create empty results\n",
    "results = anlr.setupResults()\n",
    "\n",
    "# ii. Fill it up for comprehensive code coverage\n",
    "assert len(RS.CLsChi2All) == 3\n",
    "clExpDetChi2 = ('detection probability', 'chi-square test probability determined expected', 'Value')\n",
    "preChi2NormalRange = [0, 0.2, 0.5, 0.8, 1]\n",
    "for preChi20 in [np.nan] + preChi2NormalRange:\n",
    "    for preChi21 in [np.nan] + ([] if np.isnan(preChi20) else preChi2NormalRange):\n",
    "        for preChi22 in [np.nan] + ([] if np.isnan(preChi21) else preChi2NormalRange):\n",
    "            expDetChi2 = preChi22 if not np.isnan(preChi22) else preChi21 if not np.isnan(preChi21) else preChi20\n",
    "            results.append(pd.Series({RS.CLsChi2All[2]: preChi20, RS.CLsChi2All[1]: preChi21,\n",
    "                                      RS.CLsChi2All[0]: preChi22, clExpDetChi2: expDetChi2}))\n",
    "            \n",
    "# iii. Post-compute Chi2\n",
    "results._postComputeChi2()\n",
    "\n",
    "# iv. Auto-check results\n",
    "assert results._dfData[RS.CLChi2].compare(results._dfData[clExpDetChi2]).empty\n",
    "logger.info('Success !')\n",
    "\n",
    "# v. Done\n",
    "#results._dfData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e. _postComputeDeltaAicDCv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1/2: normal case\n",
    "\n",
    "# i. Create empty results\n",
    "results = anlr.setupResults()\n",
    "\n",
    "# ii. Fill it up for comprehensive code coverage\n",
    "clSampEsp = ('header (sample)',  'Espèce', 'Value')\n",
    "clSampPas = ('header (sample)', 'Passage', 'Value')\n",
    "clSampAdl = ('header (sample)',  'Adulte', 'Value')\n",
    "clSampDur = ('header (sample)',   'Durée', 'Value')\n",
    "ldSamples = [{clSampEsp: 'Wood troobidoo', clSampPas: '99', clSampAdl: 'Yes', clSampDur: '1 century'},\n",
    "             {clSampEsp: 'Garden screamer', clSampPas: '374', clSampAdl: 'May be', clSampDur: '2 micro-seconds'}]\n",
    "\n",
    "clExpDeltaAic = ('detection probability', 'Delta AIC expected', 'Value')\n",
    "clExpDeltaDcv = ('density/abundance', 'density of animals expected', 'Delta Cv')\n",
    "dExpDeltaAic = {np.nan: np.nan, 200: 0, 1000: 800}\n",
    "dExpDeltaDcv = {np.nan: np.nan, 0.1: 0, 1: 0.9}\n",
    "\n",
    "for dSample in ldSamples:\n",
    "    for leftTrDist in [np.nan, 10]:\n",
    "        dSample[RS.CLsTruncDist[0]] = leftTrDist\n",
    "        for rightTrDist in [np.nan, 200]:\n",
    "            dSample[RS.CLsTruncDist[1]] = rightTrDist\n",
    "            for aic in [np.nan, 200, 1000]:\n",
    "                for dcv in [np.nan, 0.1, 1]:\n",
    "                    results.append(pd.Series({RS.CLAic: aic, clExpDeltaAic: dExpDeltaAic[aic],\n",
    "                                              RS.CLDCv: dcv, clExpDeltaDcv: dExpDeltaDcv[dcv]}),\n",
    "                                   sCustomHead=pd.Series(dSample))\n",
    "\n",
    "# iii. Post-compute Delta AIC and DeltaDCv\n",
    "results._postComputeDeltaAicDCv()\n",
    "\n",
    "# iv. Auto-check results\n",
    "assert results._dfData[RS.CLDeltaAic].compare(results._dfData[clExpDeltaAic]).empty\n",
    "assert results._dfData[RS.CLDeltaDCv].compare(results._dfData[clExpDeltaDcv]).empty\n",
    "logger.info('Success !')\n",
    "\n",
    "# v. Done\n",
    "#results._dfData  # .to_excel('tmp/_.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2/2: special case with one all-NaN sample id column\n",
    "\n",
    "# i. Create empty results\n",
    "results = anlr.setupResults()\n",
    "\n",
    "# ii. Fill it up for comprehensive code coverage\n",
    "clSampEsp = ('header (sample)',  'Espèce', 'Value')\n",
    "clSampPas = ('header (sample)', 'Passage', 'Value')\n",
    "clSampAdl = ('header (sample)',  'Adulte', 'Value')\n",
    "clSampDur = ('header (sample)',   'Durée', 'Value')\n",
    "ldSamples = [{clSampEsp: 'Wood troobidoo', clSampPas: np.nan, clSampAdl: 'Yes', clSampDur: '1 century'},\n",
    "             {clSampEsp: 'Garden screamer', clSampPas: np.nan, clSampAdl: 'May be', clSampDur: '2 micro-seconds'}]\n",
    "\n",
    "clExpDeltaAic = ('detection probability', 'Delta AIC expected', 'Value')\n",
    "clExpDeltaDcv = ('density/abundance', 'density of animals expected', 'Delta Cv')\n",
    "dExpDeltaAic = {np.nan: np.nan, 200: 0, 1000: 800}\n",
    "dExpDeltaDcv = {np.nan: np.nan, 0.1: 0, 1: 0.9}\n",
    "\n",
    "for dSample in ldSamples:\n",
    "    for leftTrDist in [np.nan, 10]:\n",
    "        dSample[RS.CLsTruncDist[0]] = leftTrDist\n",
    "        for rightTrDist in [np.nan, 200]:\n",
    "            dSample[RS.CLsTruncDist[1]] = rightTrDist\n",
    "            for aic in [np.nan, 200, 1000]:\n",
    "                for dcv in [np.nan, 0.1, 1]:\n",
    "                    results.append(pd.Series({RS.CLAic: aic, clExpDeltaAic: dExpDeltaAic[aic],\n",
    "                                              RS.CLDCv: dcv, clExpDeltaDcv: dExpDeltaDcv[dcv]}),\n",
    "                                   sCustomHead=pd.Series(dSample))\n",
    "\n",
    "# iii. Post-compute Delta AIC and DeltaDCv\n",
    "results._postComputeDeltaAicDCv()\n",
    "\n",
    "# iv. Auto-check results\n",
    "assert results._dfData[RS.CLDeltaAic].compare(results._dfData[clExpDeltaAic]).empty\n",
    "assert results._dfData[RS.CLDeltaDCv].compare(results._dfData[clExpDeltaDcv]).empty\n",
    "logger.info('Success !')\n",
    "\n",
    "# v. Done\n",
    "results._dfData  # .to_excel('tmp/_2.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### f. _postComputeQualityIndicators (and callees)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# 1/2 Normal case\n",
    "\n",
    "# i. Create empty results\n",
    "results = anlr.setupResults()\n",
    "\n",
    "# ii. Fill it up for comprehensive code coverage\n",
    "statsRange = [np.nan, 0, 0.5, 1]  # Range for statistic tests\n",
    "densCvRange = [np.nan, 0.1, 0.3, 1]  # Range for density coef. of variation\n",
    "\n",
    "logger.info('Filling up results to post-compute')\n",
    "ldRes = list()\n",
    "\n",
    "# Try all key functions\n",
    "for keyFn in [np.nan, 'HNORMAL', 'UNIFORM', 'HAZARD', 'NEXPON']:\n",
    "    # CLNTotPars and CLNAdjPars are used independently, no need for combinations)\n",
    "    for nPars in [np.nan, 0, 2, 10]:\n",
    "        # CLNObs and CLNTotObs are always used together, no need for full combination stuff\n",
    "        for nObs, nTotObs in [(np.nan, 50), (10, np.nan), (25, 50), (50, 50)]:\n",
    "            for chi2 in statsRange:\n",
    "                for ks in statsRange:\n",
    "                    for cvmUw in statsRange:\n",
    "                        for cvmCw in statsRange:\n",
    "                            for dcv in densCvRange:\n",
    "                                # Version 1. results.append(pd.Series(...)) \n",
    "                                # results.append(pd.Series(...))\n",
    "                                \n",
    "                                # Version 2 (1/2). Save dict to list for later\n",
    "                                ldRes.append({RS.CLKeyFn: keyFn, RS.CLNAdjPars: nPars, RS.CLNTotPars:nPars,\n",
    "                                              RS.CLNObs: nObs, RS.CLNTotObs: nTotObs,\n",
    "                                              RS.CLChi2: chi2, RS.CLKS: ks,\n",
    "                                              RS.CLCvMUw: cvmUw, RS.CLCvMCw: cvmCw, RS.CLDCv: dcv})\n",
    "\n",
    "# Version 1. Very slow : 14mn for 65536 results\n",
    "\n",
    "# Version 2 (2/2). Build final DataFrame in 1 fast operation\n",
    "# => very fast : total 260ms for 65536 results !!!\n",
    "results._dfData = pd.DataFrame(ldRes)\n",
    "results._dfData.columns = pd.MultiIndex.from_tuples(results._dfData.columns)\n",
    "\n",
    "logger.info('{} results'.format(len(results._dfData)))\n",
    "\n",
    "# iii. Post-compute Delta AIC and DeltaDCv\n",
    "results._postComputeQualityIndicators()\n",
    "\n",
    "# Note: For 65536 results, Windows 10, total elapsed :\n",
    "# * 2021-11-21 : i5-8365U : ~20-25mn (unoptimized pd.DataFrame.apply(axis='columns')-based code)\n",
    "# * 2021-11-28 : i7-10850H : ~370ms ! (numpy-optimized column-array-operation-based code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iv. Auto-check results (statistically, not always value by value)\n",
    "df = results._dfData\n",
    "\n",
    "# SightRate\n",
    "assert df[RS.CLSightRate].isnull().sum() == 40960\n",
    "assert df[RS.CLSightRate].value_counts().eq(pd.Series({100: 20480, 50: 20480})).all()\n",
    "\n",
    "# New quality indicators that should be killed by at least 1 NaN in source results\n",
    "miCompCols = [RS.CLKeyFn, RS.CLNAdjPars, RS.CLNTotPars, RS.CLNObs, RS.CLNTotObs,\n",
    "              RS.CLChi2, RS.CLKS, RS.CLCvMUw, RS.CLCvMCw, RS.CLDCv]\n",
    "miNewQuaIndCols = [RS.CLCmbQuaBal2, RS.CLCmbQuaBal3, RS.CLCmbQuaChi2, RS.CLCmbQuaKS, RS.CLCmbQuaDCv]\n",
    "dfNaNKilledNewQuaIndics = df.loc[df[miCompCols].isnull().any(axis='columns'), miNewQuaIndCols]\n",
    "aHist = np.histogram(dfNaNKilledNewQuaIndics, bins=np.linspace(0, 0.15, 16))[0]\n",
    "logger.info('Histogram of NaN-killed new Qua Indics: ' + str(aHist))\n",
    "\n",
    "assert pd.Series(aHist).eq([378737, 423, 0, 2, 64, 189, 292, 245, 202, 120, 118, 40, 8, 0, 0]).all()\n",
    "\n",
    "# All quality indicators that were not killed by any NaN in source results\n",
    "miQuaIndCols = [RS.CLCmbQuaBal1] + miNewQuaIndCols\n",
    "dfNoNaNQuaIndics = df.loc[df[miCompCols].notnull().all(axis='columns'), miQuaIndCols]\n",
    "aHist = np.histogram(dfNoNaNQuaIndics, bins=np.linspace(0, 1, 11))[0]\n",
    "logger.info('Histogram of no-NaN Qua Indics: ' + str(aHist))\n",
    "\n",
    "assert pd.Series(aHist).eq([30788, 876, 0, 77, 403, 800, 918, 735, 339, 56]).all()\n",
    "\n",
    "# All quality indicators that were not killed by any NaN in source results\n",
    "# or 0 in source stat tests results or bad DCv or SightRate or NAdjustParams\n",
    "miStatestCols = [RS.CLChi2, RS.CLKS, RS.CLCvMUw, RS.CLCvMCw]\n",
    "dfNotSoBadQuaIndics = df.loc[df[miCompCols].notnull().all(axis='columns')\n",
    "                             & df[miStatestCols].gt(0.01).all(axis='columns')\n",
    "                             & df[RS.CLDCv].lt(0.3) & df[RS.CLNAdjPars].lt(4)\n",
    "                             & (df[RS.CLNObs] / df[RS.CLNTotObs]).gt(0.7), miQuaIndCols]\n",
    "aHist = np.histogram(dfNotSoBadQuaIndics, bins=np.linspace(0.3, 1, 8))[0]\n",
    "logger.info('Histogram of not-so-bad Qua Indics: ' + str(aHist))\n",
    "assert pd.Series(aHist).eq([0, 4, 60, 174, 256, 224, 50]).all()\n",
    "\n",
    "logger.info('Success !')\n",
    "\n",
    "# v. Done\n",
    "results._dfData  # .to_excel('tmp/_2.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2/2 Special case, with 1 missing input column, let's say CLCvMUw\n",
    "# Note: From far faster, only 72 results to compute\n",
    "\n",
    "# i. Create empty results\n",
    "results = anlr.setupResults()\n",
    "\n",
    "# ii. Fill it up\n",
    "statestRange = [0, 0.5, 1]  # Range for statistic tests\n",
    "densCvRange = [0.1, 0.3]  # Range for density coef. of variation\n",
    "\n",
    "logger.info('Filling up results to post-compute')\n",
    "ldRes = list()\n",
    "nObs, nTotObs = 45, 50  # Make tests last less (already tests before)\n",
    "\n",
    "# Try all key functions\n",
    "for keyFn in ['HNORMAL', 'UNIFORM', 'HAZARD', 'NEXPON']:\n",
    "    # CLNTotPars and CLNAdjPars are used independently, no need for combinations)\n",
    "    for nPars in [0, 1, 2]:\n",
    "        # CLNObs and CLNTotObs are always used together, no need for full combination stuff\n",
    "        for chi2 in statestRange:\n",
    "            ks = cvmCw = chi2  # Make tests last less (already tests before)\n",
    "            for dcv in densCvRange:\n",
    "                ldRes.append({RS.CLKeyFn: keyFn, RS.CLNAdjPars: nPars, RS.CLNTotPars:nPars,\n",
    "                              RS.CLNObs: nObs, RS.CLNTotObs: nTotObs,\n",
    "                              RS.CLChi2: chi2, RS.CLKS: ks,\n",
    "                              RS.CLCvMCw: cvmCw, RS.CLDCv: dcv})  # Note: RS.CLCvMUw is missing\n",
    "\n",
    "results._dfData = pd.DataFrame(ldRes)\n",
    "results._dfData.columns = pd.MultiIndex.from_tuples(results._dfData.columns)\n",
    "\n",
    "logger.info('{} results'.format(len(results._dfData)))\n",
    "\n",
    "# iii. Post-compute Delta AIC and DeltaDCv\n",
    "results._postComputeQualityIndicators()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iv. Auto-check results (statistically, not value by value)\n",
    "df = results._dfData\n",
    "\n",
    "# New quality indicators that should be killed because of at least 1 NaN in source results (the missing CLCvMUw)\n",
    "miNewQuaIndCols = [RS.CLCmbQuaBal2, RS.CLCmbQuaBal3, RS.CLCmbQuaChi2, RS.CLCmbQuaKS, RS.CLCmbQuaDCv]\n",
    "assert df[miNewQuaIndCols].eq(0).all().all()\n",
    "\n",
    "logger.info('Success !')\n",
    "\n",
    "# v. Done\n",
    "results._dfData  # .to_excel('tmp/_3.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### g. _sampleDistTruncGroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "# Note: Okay, it's actually an MCDSTruncOptAnalysisResultsSet file ... but we'll ignore the extra columns, promised :-)\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "results = anlr.setupResults()\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get results table and extract only needed columns (promise fulfilled ;-)\n",
    "# (Note: No post-computed column used here, so ... recomputation authorised, but not needed, and slower)\n",
    "CLSampNum = ('header (head)', 'NumEchant', 'Value')\n",
    "dfRes = results.getData(copy=True)[[CLSampNum, RS.CLParTruncLeft, RS.CLParTruncRight]]\n",
    "dfRes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test case: a hard-coded extract of once refin/ACDC2019-Naturalist-UnitestOptResultats.ods results file\n",
    "#CLSampNum = ('header (head)', 'NumEchant', 'Value')\n",
    "#dfRes = pd.DataFrame(index=[18, 17, 5, 16, 15, 0, 12, 8, 14, 9, 1, 19, 2, 22, 24, 21, 20, 26, 23, 27, 28, 30, 29, 32, 31,\n",
    "#                            33, 35, 34, 37, 36, 40, 41, 39, 43, 44, 47, 48, 49, 50, 52, 51, 56, 53, 54, 57, 60, 62, 69,\n",
    "#                            63, 65, 67, 68, 66, 64],\n",
    "#                     columns=pd.MultiIndex.from_tuples([CLSampNum, RS.CLParTruncLeft, RS.CLParTruncRight]),\n",
    "#                     data=[[0, np.nan, np.nan], [0, 10.0, np.nan], [0, np.nan, 280.0], [0, 15.0, 300.0], [0, 20.0, 300.0],\n",
    "#                           [0, 12.9, 367.2], [0, np.nan, np.nan], [0, 10.0, np.nan], [0, np.nan, 280.0], [0, 15.0, 300.0],\n",
    "#                           [0, 20.0, 300.0], [0, 24.1, 229.8],\n",
    "#                           [1, np.nan, np.nan], [1, 10.0, np.nan], [1, np.nan, 280.0], [1, 15.0, 300.0], [1, 20.0, 300.0],\n",
    "#                           [1, np.nan, np.nan], [1, 10.0, np.nan], [1, np.nan, 280.0], [1, 15.0, 300.0], [1, 20.0, 300.0],\n",
    "#                           [2, np.nan, np.nan], [2, np.nan, 450.0], [2, np.nan, 500.0], [2, np.nan, np.nan],\n",
    "#                           [2, np.nan, 450.0], [2, np.nan, 500.0],\n",
    "#                           [3, np.nan, np.nan], [3, np.nan, 500.0], [3, np.nan, 600.0], [3, 17.2, 200.0], [3, np.nan, np.nan],\n",
    "#                           [3, np.nan, 500.0], [3, np.nan, 600.0], [3, 4.4, 200.0],\n",
    "#                           [4, np.nan, np.nan], [4, 10.0, np.nan], [4, np.nan, 280.0], [4, 15.0, 300.0], [4, np.nan, 731.1],\n",
    "#                           [4, np.nan, np.nan], [4, 10.0, np.nan], [4, np.nan, 280.0], [4, 15.0, 300.0], [4, np.nan, 477.6],\n",
    "#                           [5, np.nan, np.nan], [5, 10.0, np.nan], [5, np.nan, 350.0], [5, 15.0, 370.0], [5, np.nan, np.nan],\n",
    "#                           [5, 10.0, np.nan], [5, np.nan, 350.0], [5, 15.0, 370.0]])\n",
    "#dfRes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Expected results per sample.\n",
    "KDSampGroupNums = \\\n",
    "{\n",
    "    0: dict(left={18: 0, 17: 1, 5: 0, 16: 2, 15: 3, 0: 1, 12: 0, 8: 1, 14: 0, 9: 2, 1: 3, 19: 3},\n",
    "            right={18: 0, 17: 0, 5: 2, 16: 2, 15: 2, 0: 3, 12: 0, 8: 0, 14: 2, 9: 2, 1: 2, 19: 1}),\n",
    "    1: dict(left={2: 0, 22: 1, 24: 0, 21: 1, 20: 2, 26: 0, 23: 1, 27: 0, 28: 1, 30: 2},\n",
    "            right={2: 0, 22: 0, 24: 1, 21: 1, 20: 1, 26: 0, 23: 0, 27: 1, 28: 1, 30: 1}),\n",
    "    2: dict(left={29: 0, 32: 0, 31: 0, 33: 0, 35: 0, 34: 0},\n",
    "            right={29: 0, 32: 1, 31: 2, 33: 0, 35: 1, 34: 2}),\n",
    "    3: dict(left={37: 0, 36: 0, 40: 0, 41: 2, 39: 0, 43: 0, 44: 0, 47: 1},\n",
    "            right={37: 0, 36: 2, 40: 3, 41: 1, 39: 0, 43: 2, 44: 3, 47: 1}),\n",
    "    4: dict(left={48: 0, 49: 1, 50: 0, 52: 1, 51: 0, 56: 0, 53: 1, 54: 0, 57: 1, 60: 0},\n",
    "            right={48: 0, 49: 0, 50: 1, 52: 1, 51: 3, 56: 0, 53: 0, 54: 1, 57: 1, 60: 2}),\n",
    "    5: dict(left={62: 0, 69: 1, 63: 0, 65: 1, 67: 0, 68: 1, 66: 0, 64: 1},\n",
    "            right={62: 0, 69: 0, 63: 1, 65: 1, 67: 0, 68: 0, 66: 1, 64: 1})\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dRefGroupNums = dict()  # For building _distTruncGroups reference :-)\n",
    "\n",
    "for lblSamp in dfRes[CLSampNum].unique():\n",
    "    \n",
    "    logger.debug(f'* {lblSamp}')\n",
    "    dSampGroupNums = RS._sampleDistTruncGroups(dfRes[dfRes[CLSampNum] == lblSamp],\n",
    "                                               ldIntrvSpecs=ldTruncIntrvSpecs, intrvEpsilon=truncIntrvEpsilon)\n",
    "    assert all(sGroupNums.eq(pd.Series(KDSampGroupNums[lblSamp][colAlias])).all()\n",
    "               for colAlias, sGroupNums in dSampGroupNums.items())\n",
    "    \n",
    "    for colAlias, sGroupNums in dSampGroupNums.items():\n",
    "        if colAlias not in dRefGroupNums:\n",
    "            dRefGroupNums[colAlias] = sGroupNums\n",
    "        else:\n",
    "            dRefGroupNums[colAlias] = dRefGroupNums[colAlias].append(sGroupNums)\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check and build results reference\n",
    "#lblSamp = 5\n",
    "#\n",
    "#df = dfRes[dfRes[CLSampNum] == lblSamp].copy()\n",
    "#\n",
    "#dGroupNums = RS._sampleDistTruncGroups(df, ldIntrvSpecs=ldIntrvSpecs, intrvEpsilon=intrvEpsilon)\n",
    "#for colAlias, sGroupNums in dGroupNums.items():\n",
    "#    df[RS.DCLGroupTruncDist[colAlias]] = sGroupNums\n",
    "#    print(colAlias, '=', sGroupNums.to_dict(), ',', sep='')\n",
    "#df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### h. _distTruncGroups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dGroupNums = results._distTruncGroups()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check\n",
    "assert all(sGroupNums.eq(dRefGroupNums[colAlias]).all() for colAlias, sGroupNums in dGroupNums.items())\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### i. _filterSortKeySchemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "results = anlr.setupResults(ldFilSorKeySchemes=None)  # Will use predefined filter-sort key generation schemes\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert results._filterSortKeySchemes() == ads.MCDSAnalysisResultsSet.AutoFilSorKeySchemes\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "ldFilSorKeySchemes = \\\n",
    "    [dict(key=RS.CLGrpOrdClTrChi2KSDCv,\n",
    "          sort=[RS.CLGroupTruncLeft, RS.CLGroupTruncRight,\n",
    "                RS.CLChi2, RS.CLKS, RS.CLDCv, RS.CLNObs, RS.CLRunStatus],\n",
    "          ascend=[True, True, False, False, True, False, True],\n",
    "          group=[RS.CLGroupTruncLeft, RS.CLGroupTruncRight]),\n",
    "     dict(key=RS.CLGblOrdDAicChi2KSDCv,\n",
    "          sort=[RS.CLParTruncLeft, RS.CLParTruncRight, RS.CLParModFitDistCuts,\n",
    "                RS.CLDeltaAic, RS.CLChi2, RS.CLKS, RS.CLDCv, RS.CLNObs, RS.CLRunStatus],\n",
    "          ascend=[True, True, True, True, False, False, True, False, True], napos='first')]\n",
    "\n",
    "results = anlr.setupResults(ldFilSorKeySchemes=ldFilSorKeySchemes)  # Will not use predefined ones.\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert results._filterSortKeySchemes() == ldFilSorKeySchemes\n",
    "\n",
    "print('Yessssss !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### j. _sampleFilterSortKeys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "# Note: Okay, it's actually an MCDSTruncOptAnalysisResultsSet file ... but we'll ignore the extra columns, promised :-)\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "results = anlr.setupResults()\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get results table and extract only needed columns (promise fulfilled ;-)\n",
    "# (Note: No post-computed column used here, so ... recomputation authorised, but not needed, and slower)\n",
    "CLSampNum = ('header (head)', 'NumEchant', 'Value')\n",
    "dfRes = results.getData(copy=True)[[CLSampNum, RS.CLParTruncLeft, RS.CLParTruncRight,\n",
    "                                    RS.CLGroupTruncLeft, RS.CLGroupTruncRight, RS.CLChi2, RS.CLDCv]]\n",
    "dfRes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Expected results per sample.\n",
    "CLGrpOrdClTrChi2 = (RS.CLCAutoFilSor, 'Chi2 (close trunc)', RS.CLTSortOrder)\n",
    "CLGblOrdDCv = (RS.CLCAutoFilSor, 'DCv (global)', RS.CLTSortOrder)\n",
    "\n",
    "KDSampFilSorKeys = \\\n",
    "{\n",
    "    0: {CLGrpOrdClTrChi2: {12: 0, 18: 1, 14: 0, 5: 1, 8: 0, 17: 1, 9: 0, 16: 1, 0: 0, 1: 0, 15: 1, 19: 0},\n",
    "        CLGblOrdDCv:      {18: 0, 12: 1, 5: 2, 14: 3, 17: 4, 8: 5, 0: 6, 16: 7, 9: 8, 15: 9, 1: 10, 19: 11}},\n",
    "    1: {CLGrpOrdClTrChi2: {26: 0, 2: 1, 27: 0, 24: 1, 23: 0, 22: 1, 28: 0, 21: 1, 30: 0, 20: 1},\n",
    "        CLGblOrdDCv:      {2: 0, 26: 1, 27: 2, 24: 3, 22: 4, 23: 5, 21: 6, 28: 7, 20: 8, 30: 9}},\n",
    "    2: {CLGrpOrdClTrChi2: {29: 0, 33: 1, 32: 0, 35: 1, 31: 0, 34: 1},\n",
    "        CLGblOrdDCv:      {29: 0, 33: 1, 32: 2, 35: 3, 31: 4, 34: 5}},\n",
    "    3: {CLGrpOrdClTrChi2: {37: 0, 39: 1, 36: 0, 43: 1, 40: 0, 44: 1, 47: 0, 41: 0},\n",
    "        CLGblOrdDCv:      {37: 0, 39: 1, 36: 2, 43: 3, 40: 4, 44: 5, 47: 6, 41: 7}},\n",
    "    4: {CLGrpOrdClTrChi2: {56: 0, 48: 1, 60: 0, 54: 1, 50: 2, 51: 0, 53: 0, 49: 1, 57: 0, 52: 1},\n",
    "        CLGblOrdDCv:      {48: 0, 56: 1, 50: 2, 54: 3, 60: 4, 51: 5, 49: 6, 53: 7, 52: 8, 57: 9}},\n",
    "    5: {CLGrpOrdClTrChi2: {62: 0, 67: 1, 63: 0, 66: 1, 69: 0, 68: 1, 64: 0, 65: 1},\n",
    "        CLGblOrdDCv:      {62: 0, 67: 1, 63: 2, 66: 3, 69: 4, 68: 5, 65: 6, 64: 7}}\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make test simpler : replace filter and sort key generation predefined scheme set by a shorter one.\n",
    "ldFilSorKeySchemes = \\\n",
    "    [dict(key=CLGrpOrdClTrChi2,\n",
    "          sort=[RS.CLGroupTruncLeft, RS.CLGroupTruncRight, RS.CLChi2],\n",
    "          ascend=[True, True, False],\n",
    "          group=[RS.CLGroupTruncLeft, RS.CLGroupTruncRight]),\n",
    "     dict(key=CLGblOrdDCv,\n",
    "          sort=[RS.CLParTruncLeft, RS.CLParTruncRight, RS.CLDCv],\n",
    "          ascend=[True, True, True], napos='first')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dRefFilSorKeys = dict()  # For building _filterSortKeys output reference :-)\n",
    "\n",
    "for lblSamp in dfRes[CLSampNum].unique():\n",
    "    \n",
    "    logger.debug(f'* {lblSamp}')\n",
    "    dSampFSKeys = RS._sampleFilterSortKeys(dfRes[dfRes[CLSampNum] == lblSamp], ldFilSorKeySchemes=ldFilSorKeySchemes)\n",
    "    assert all(sFSKeys.eq(pd.Series(KDSampFilSorKeys[lblSamp][colLbl])).all()\n",
    "               for colLbl, sFSKeys in dSampFSKeys.items())\n",
    "    \n",
    "    for colLbl, sFSKeys in dSampFSKeys.items():\n",
    "        if colLbl not in dRefFilSorKeys:\n",
    "            dRefFilSorKeys[colLbl] = sFSKeys\n",
    "        else:\n",
    "            dRefFilSorKeys[colLbl] = dRefFilSorKeys[colLbl].append(sFSKeys)\n",
    "\n",
    "print('Yesssssss !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check (by eyes) and build material for results reference\n",
    "#lblSamp = 5\n",
    "#df = dfRes[dfRes[CLSampNum] == lblSamp].copy()\n",
    "#\n",
    "#dSampFSKeys = RS._sampleFilterSortKeys(df, ldFilSorKeySchemes=ldFilSorKeySchemes)\n",
    "#for colLbl, sFSKeys in dSampFSKeys.items():\n",
    "#    df[colLbl] = sFSKeys\n",
    "#    print(sFSKeys.to_dict(), ',', sep='')\n",
    "#\n",
    "#display(df.sort_values(by=[RS.CLGroupTruncLeft, RS.CLGroupTruncRight, RS.CLChi2],\n",
    "#                       ascending=[True, True, False]) \\\n",
    "#         [[RS.CLGroupTruncLeft, RS.CLGroupTruncRight, RS.CLChi2, CLGrpOrdClTrChi2]])\n",
    "#display(df.sort_values(by=[RS.CLParTruncLeft, RS.CLParTruncRight, RS.CLDCv],\n",
    "#                       ascending=[True, True, True], na_position='first') \\\n",
    "#         [[RS.CLParTruncLeft, RS.CLParTruncRight, RS.CLDCv, CLGblOrdDCv]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### k. _filterSortKeys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "# Note: Okay, it's actually an MCDSTruncOptAnalysisResultsSet file ... but we'll ignore the extra columns, promised :-)\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "# Make test simpler : replace filter and sort key predefined generation scheme set by a shorter one.\n",
    "results = anlr.setupResults(ldFilSorKeySchemes=ldFilSorKeySchemes)  # See j. right above for ldFilSorKeySchemes definition  !\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get results table and extract only needed columns (promise fulfilled ;-)\n",
    "# (Note: No post-computed column used here, so ... recomputation authorised, but not needed, and slower)\n",
    "CLSampNum = ('header (head)', 'NumEchant', 'Value')\n",
    "dfRes = results.getData(copy=True)[[CLSampNum, RS.CLParTruncLeft, RS.CLParTruncRight,\n",
    "                                    RS.CLGroupTruncLeft, RS.CLGroupTruncRight, RS.CLChi2, RS.CLDCv]]\n",
    "dfRes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dFilSorKeys = results._filterSortKeys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check\n",
    "assert all(sFSKeys.eq(dRefFilSorKeys[colLbl]).all() for colLbl, sFSKeys in dFilSorKeys.items())\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### l. _indexOfDuplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test cases\n",
    "df = pd.DataFrame([dict(a=1.000, b=2.00, c=3.0, d='To be kept: first so as a.round(1) == 1.0, whatever c, b == 2'),\n",
    "                   dict(a=1.010, b=2.00, c=1.0, d='Duplicate: 2nd so as a.round(1) == 1.0, whatever c, b == 2'),\n",
    "                   dict(a=1.049, b=2.00, c=2.0, d='Duplicate: 3rd so as a.round(1) == 1.0, whatever c, b == 2'),\n",
    "                   dict(a=1.051, b=2.00, c=2.0, d='To be kept: first so as a.round(1) == 1.1, whatever c, b == 2'),\n",
    "                   dict(a=1.060, b=2.00, c=2.0, d='Duplicate: 2nd so as a.round(1) == 1.1, whatever c, b == 2'),\n",
    "                   dict(a=1.100, b=2.00, c=4.0, d='Duplicate: 3rd so as a.round(1) == 1.1, whatever c, b == 2'),\n",
    "                   dict(a=1.151, b=2.00, c=5.0, d='To be kept: first so as a.round(1) == 1.2, whatever c, b == 2'),\n",
    "                   dict(a=2.000, b=2.00, c=3.0, d='To be kept: first so as b == 2.0, whatever c, a == 2'),\n",
    "                   dict(a=2.000, b=2.00, c=5.0, d='Duplicate: 2nd so as b == 2.0, whatever c, a == 2'),\n",
    "                   dict(a=2.000, b=2.01, c=9.0, d='To be kept: first so as b == 2.0, whatever c, a == 2'),\n",
    "                   dict(a=2.000, b=1.9999999, c=3.0, d='To be kept: first so as b == 1.9999999, whatever c, a == 2')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute filter\n",
    "iDupes = RS._indexOfDuplicates(df, keep='first', subset=['a', 'b'], round2decs=dict(a=1))\n",
    "iDupes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply filter\n",
    "df.drop(iDupes, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check\n",
    "assert all(iDupes == [1, 2, 4, 5, 8])\n",
    "assert all('Duplicate' not in s for s in df.d)\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### m. _indexOfWorstOneCriterion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test cases\n",
    "df = pd.DataFrame([dict(s=0, a=1.000),\n",
    "                   dict(s=0, a=0.010),\n",
    "                   dict(s=0, a=1.049),\n",
    "                   dict(s=0, a=1.051),\n",
    "                   dict(s=0, a=0.060),\n",
    "                   dict(s=0, a=1.100),\n",
    "                   dict(s=0, a=1.151),\n",
    "                   dict(s=0, a=2.000),\n",
    "                   dict(s=0, a=1.020),\n",
    "                   dict(s=0, a=1.500),\n",
    "                   dict(s=0, a=2.000),\n",
    "                   dict(s=0, a=1.010),\n",
    "                   dict(s=0, a=1.049),\n",
    "                   dict(s=0, a=0.051),\n",
    "\n",
    "                   dict(s=1, a=1.060),\n",
    "                   dict(s=1, a=1.100),\n",
    "                   dict(s=1, a=1.151),\n",
    "\n",
    "                   dict(s=2, a=3.000),\n",
    "                   dict(s=2, a=2.000),\n",
    "                   dict(s=2, a=6.000),\n",
    "                   dict(s=2, a=0.060),\n",
    "                   dict(s=2, a=1.100),\n",
    "                   dict(s=2, a=3.010),\n",
    "                   dict(s=2, a=2.200),\n",
    "                   dict(s=2, a=2.230),\n",
    "\n",
    "                   dict(s=3, a=1.100),\n",
    "                   dict(s=3, a=1.151),\n",
    "                   dict(s=3, a=2.000),\n",
    "                   dict(s=3, a=2.000),\n",
    "\n",
    "                   dict(s=4, a=2.000),\n",
    "                   dict(s=4, a=2.000),\n",
    "                   dict(s=4, a=2.000),\n",
    "                   dict(s=4, a=2.000),\n",
    "                   dict(s=4, a=2.000),\n",
    "                   dict(s=4, a=2.000)])\n",
    "\n",
    "s2filter = [0, 2, 3, 5]  # Ignore sample 1 and 4, add empty sample 5\n",
    "\n",
    "maxRes = 6  # Keep 6 best values at most."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.s.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.sort_values(by=['s', 'a'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute filter\n",
    "i2drop = RS._indexOfWorstOneCriterion(df, sampleIds=s2filter, sampleIdCol='s', critCol='a', ascendCrit=False, nTgtRes=maxRes)\n",
    "\n",
    "i2drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Apply filter\n",
    "df.drop(i2drop, inplace=True)\n",
    "\n",
    "df.sort_values(by=['s', 'a'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.s.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check\n",
    "assert all(i2drop == [2, 12, 8, 11, 0, 4, 13, 1, 21, 20])\n",
    "assert df[df.s.isin(s2filter)].s.value_counts().le(maxRes).all()\n",
    "assert df.loc[df.s.isin(s2filter)].groupby('s').a.max().le([2, 6, 2]).all()\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### n. _indexOfWorstMultiOrderCriteria"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test cases\n",
    "df = pd.DataFrame([dict(s=0, a=1, b=1, c='Kept thanks to a and b'),\n",
    "                   dict(s=0, a=0, b=1, c='Kept thanks to a and b'),\n",
    "                   dict(s=0, a=2, b=2, c='Dropped because of a and b'),\n",
    "                   dict(s=0, a=4, b=3, c='Dropped because of a and b'),\n",
    "                   dict(s=0, a=3, b=2, c='Dropped because of a and b'),\n",
    "                   dict(s=0, a=5, b=1, c='Kept thanks to b'),\n",
    "                   dict(s=1, a=2, b=4, c='Dropped because of a and b'),\n",
    "                   dict(s=1, a=1, b=3, c='Kept thanks to a'),\n",
    "                   dict(s=1, a=4, b=0, c='Kept thanks to b')])\n",
    "\n",
    "critCols = ['a', 'b']\n",
    "supCrit = 2\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "i2drop = RS._indexOfWorstMultiOrderCriteria(df, critCols=critCols, supCrit=supCrit)\n",
    "\n",
    "i2drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply filter\n",
    "df.drop(i2drop, inplace=True)\n",
    "\n",
    "df.sort_values(by=['s', 'a'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check\n",
    "assert all(i2drop == [2, 3, 4, 6])\n",
    "assert all('Dropped' not in s for s in df.c)\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### o. filSorSchemeId"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RS = ads.MCDSAnalysisResultsSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fsRes = ads.MCDSAnalysisResultsSet(sampleIndCol='Sample')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dupSubset = [RS.CLNObs, RS.CLEffort, RS.CLDeltaAic, RS.CLChi2, RS.CLKS, RS.CLCvMUw, RS.CLCvMCw, RS.CLDCv, \n",
    "             RS.CLPDetec, RS.CLPDetecMin, RS.CLPDetecMax, RS.CLDensity, RS.CLDensityMin, RS.CLDensityMax]\n",
    "dDupRounds = {RS.CLDeltaAic: 1, RS.CLChi2: 2, RS.CLKS: 2, RS.CLCvMUw: 2, RS.CLCvMCw: 2, RS.CLDCv: 2, \n",
    "              RS.CLPDetec: 3, RS.CLPDetecMin: 3, RS.CLPDetecMax: 3, RS.CLDensity: 2, RS.CLDensityMin: 2, RS.CLDensityMax: 2}\n",
    "\n",
    "schEx = dict(method=RS.filterSortOnExecCode,\n",
    "             deduplicate=dict(dupSubset=dupSubset, dDupRounds=dDupRounds),\n",
    "             filterSort=dict(whichFinalQua=RS.CLCmbQuaBal1, ascFinalQua=False))\n",
    "\n",
    "schACCQ1 = dict(method=RS.filterSortOnExCAicMulQua,\n",
    "                deduplicate=dict(dupSubset=dupSubset, dDupRounds=dDupRounds),\n",
    "                filterSort=dict(sightRate=92.5, nBestAIC=3, nBestQua=1, \n",
    "                                whichBestQua=[RS.CLGrpOrdClTrChi2KSDCv, RS.CLGrpOrdClTrDCv, RS.CLGrpOrdClTrQuaBal1,\n",
    "                                              RS.CLGrpOrdClTrQuaChi2, RS.CLGrpOrdClTrQuaKS, RS.CLGrpOrdClTrQuaDCv],\n",
    "                                nFinalRes=12, whichFinalQua=RS.CLCmbQuaBal1, ascFinalQua=False))\n",
    "    \n",
    "schACCQ2 = copy.deepcopy(schACCQ1)\n",
    "\n",
    "schACCQ3 = copy.deepcopy(schACCQ1)\n",
    "schACCQ3['filterSort']['sightRate'] = 93.0\n",
    "\n",
    "schACCQ4 = copy.deepcopy(schACCQ1)\n",
    "schACCQ4['filterSort']['nBestAIC'] = 2\n",
    "\n",
    "schACCQ5 = copy.deepcopy(schACCQ2)\n",
    "schACCQ5['filterSort']['nBestQua'] = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert fsRes.filSorSchemeId(schEx) == 'ExCode'\n",
    "assert fsRes.filSorSchemeId(schACCQ1).startswith('ExAicMQua')\n",
    "assert fsRes.filSorSchemeId(schACCQ2) == fsRes.filSorSchemeId(schACCQ1)\n",
    "assert fsRes.filSorSchemeId(schACCQ3) != fsRes.filSorSchemeId(schACCQ1) \\\n",
    "       and fsRes.filSorSchemeId(schACCQ3).startswith('ExAicMQua')\n",
    "assert fsRes.filSorSchemeId(schACCQ4).startswith(fsRes.filSorSchemeId(schACCQ1))\n",
    "assert fsRes.filSorSchemeId(schACCQ5).startswith(fsRes.filSorSchemeId(schACCQ1))\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict(schEx=fsRes.filSorSchemeId(schEx), schACCQ1=fsRes.filSorSchemeId(schACCQ1),\n",
    "     schACCQ2=fsRes.filSorSchemeId(schACCQ2), schACCQ3=fsRes.filSorSchemeId(schACCQ3),\n",
    "     schACCQ4=fsRes.filSorSchemeId(schACCQ4), schACCQ5=fsRes.filSorSchemeId(schACCQ5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 14. MCDSTruncOptAnalysisResultsSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = ads.logger('ads.onr', level=ads.DEBUG3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RS = ads.MCDSTruncOptanalysisResultsSet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Prepare stuff for creating MCDSTruncOptanalysisResultsSet objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source / Results data\n",
    "transectPlaceCols = ['Point']\n",
    "passIdCol = 'Passage'\n",
    "effortCol = 'Effort'\n",
    "\n",
    "sampleDistCol = 'Distance'\n",
    "sampleDecCols = [effortCol, sampleDistCol]\n",
    "\n",
    "sampleNumCol = 'NumEchant'\n",
    "sampleSelCols = ['Espèce', passIdCol, 'Adulte', 'Durée']\n",
    "\n",
    "sampleAbbrevCol = 'AbrevEchant'\n",
    "\n",
    "optIndCol = 'IndOptim'\n",
    "optAbbrevCol = 'AbrevOptim'\n",
    "\n",
    "dSurveyArea = dict(Zone='ACDC', Surface='2400')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# General DS analysis parameters\n",
    "varIndCol = 'NumAnlys'\n",
    "anlysAbbrevCol = 'AbrevAnlys'\n",
    "anlysParamCols = ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "\n",
    "distanceUnit = 'Meter'\n",
    "areaUnit = 'Hectare'\n",
    "surveyType = 'Point'\n",
    "distanceType = 'Radial'\n",
    "clustering = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results post-computation parameters\n",
    "ldTruncIntrvSpecs = [dict(col='left', minDist=5.0, maxLen=5.0),\n",
    "                     dict(col='right', minDist=25.0, maxLen=25.0)]\n",
    "truncIntrvEpsilon = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load individualised observations and actual transects\n",
    "indivObsFile = 'refin/ACDC2019-Naturalist-extrait-ObsIndiv.ods'\n",
    "\n",
    "dfObsIndiv = ads.DataSet(indivObsFile, sheet='DonnéesIndiv').dfData\n",
    "\n",
    "dfTransects = ads.DataSet(indivObsFile, sheet='Inventaires').dfData\n",
    "\n",
    "dict(indivObs=len(dfObsIndiv), transects=len(dfTransects))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# What's better to create an MCDS(Opt)AnalysisResultsSet objets than a MCDSTruncationOptanalyser instance ?\n",
    "optanlr = \\\n",
    "    ads.MCDSTruncationOptanalyser(dfObsIndiv, dfTransects=dfTransects, dSurveyArea=dSurveyArea, \n",
    "                                  transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                                  sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                                  sampleDistCol=sampleDistCol,\n",
    "                                  abbrevCol=anlysAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                                  anlysIndCol=varIndCol, sampleIndCol=sampleNumCol,\n",
    "                                  distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                                  surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                                  resultsHeadCols=dict(before=[varIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                                       after=anlysParamCols + [anlysAbbrevCol]),\n",
    "                                  ldTruncIntrvSpecs=ldTruncIntrvSpecs, truncIntrvEpsilon=truncIntrvEpsilon)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. _sampleDistTruncGroups\n",
    "\n",
    "(this one is specialized from MCDSAnalysisResultsSet's, so it's NOT the same)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "results = optanlr.setupResults()\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get results table and extract only needed columns\n",
    "# (Note: No post-computed column used here, so ... recomputation authorised, but not needed, and slower)\n",
    "CLSampNum = ('header (head)', 'NumEchant', 'Value')\n",
    "dfRes = results.getData(copy=True)[[CLSampNum, RS.CLOptimTruncFlag, RS.CLParTruncLeft, RS.CLParTruncRight]]\n",
    "dfRes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Expected results per sample.\n",
    "KDSampGroupNums = \\\n",
    "{\n",
    "    0: dict(left={18: 0, 17: 1, 5: 0, 16: 1, 15: 2, 12: 0, 8: 1, 14: 0, 9: 1, 1: 2, 0: 1, 19: 2},\n",
    "            right={18: 0, 17: 0, 5: 1, 16: 1, 15: 1, 12: 0, 8: 0, 14: 1, 9: 1, 1: 1, 0: 2, 19: 1}),\n",
    "    1: dict(left={2: 0, 22: 1, 24: 0, 21: 1, 20: 2, 26: 0, 23: 1, 27: 0, 28: 1, 30: 2},\n",
    "            right={2: 0, 22: 0, 24: 1, 21: 1, 20: 1, 26: 0, 23: 0, 27: 1, 28: 1, 30: 1}),\n",
    "    2: dict(left={29: 0, 32: 0, 31: 0, 33: 0, 35: 0, 34: 0},\n",
    "            right={29: 0, 32: 1, 31: 2, 33: 0, 35: 1, 34: 2}),\n",
    "    3: dict(left={37: 0, 36: 0, 40: 0, 39: 0, 43: 0, 44: 0, 41: 2, 47: 1},\n",
    "            right={37: 0, 36: 1, 40: 2, 39: 0, 43: 1, 44: 2, 41: 1, 47: 1}),\n",
    "    4: dict(left={48: 0, 49: 1, 50: 0, 52: 1, 56: 0, 53: 1, 54: 0, 57: 1, 51: 0, 60: 0},\n",
    "            right={48: 0, 49: 0, 50: 1, 52: 1, 56: 0, 53: 0, 54: 1, 57: 1, 51: 2, 60: 1}),\n",
    "    5: dict(left={62: 0, 69: 1, 63: 0, 65: 1, 67: 0, 68: 1, 66: 0, 64: 1},\n",
    "            right={62: 0, 69: 0, 63: 1, 65: 1, 67: 0, 68: 0, 66: 1, 64: 1})\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dRefGroupNums = dict()  # For building _distTruncGroups reference :-)\n",
    "\n",
    "for lblSamp in dfRes[CLSampNum].unique():\n",
    "    \n",
    "    logger.debug(f'* {lblSamp}')\n",
    "    dSampGroupNums = RS._sampleDistTruncGroups(dfRes[dfRes[CLSampNum] == lblSamp],\n",
    "                                               ldIntrvSpecs=ldTruncIntrvSpecs, intrvEpsilon=truncIntrvEpsilon)\n",
    "    assert all(sGroupNums.eq(pd.Series(KDSampGroupNums[lblSamp][colAlias])).all()\n",
    "               for colAlias, sGroupNums in dSampGroupNums.items())\n",
    "    \n",
    "    for colAlias, sGroupNums in dSampGroupNums.items():\n",
    "        if colAlias not in dRefGroupNums:\n",
    "            dRefGroupNums[colAlias] = sGroupNums\n",
    "        else:\n",
    "            dRefGroupNums[colAlias] = dRefGroupNums[colAlias].append(sGroupNums)\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check and build results reference\n",
    "#lblSamp = 5\n",
    "#\n",
    "#df = dfRes[dfRes[CLSampNum] == lblSamp].copy()\n",
    "#\n",
    "#dGroupNums = RS._sampleDistTruncGroups(df, ldIntrvSpecs=ldTruncIntrvSpecs, intrvEpsilon=truncIntrvEpsilon)\n",
    "#for colAlias, sGroupNums in dGroupNums.items():\n",
    "#    df[RS.DCLGroupTruncDist[colAlias]] = sGroupNums\n",
    "#    print(colAlias, '=', sGroupNums.to_dict(), ',', sep='')\n",
    "#df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. _distTruncGroups\n",
    "\n",
    "Well, no test needed actually, as not any specialized from MCDSAnalysisREsultsSet's ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dGroupNums = results._distTruncGroups()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check\n",
    "assert all(sGroupNums.eq(dRefGroupNums[colAlias]).all() for colAlias, sGroupNums in dGroupNums.items())\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. _filterSortKeySchemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "results = optanlr.setupResults(ldFilSorKeySchemes=None)  # Will use predefined filter-sort key generation schemes\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert results._filterSortKeySchemes() == ads.MCDSTruncOptanalysisResultsSet.AutoFilSorKeySchemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "ldFilSorKeySchemes = \\\n",
    "    [dict(key=RS.CLGrpOrdClTrChi2KSDCv,  # Best Chi2 & KS & DCv inside groups of close truncation params\n",
    "          sort=[RS.CLOptimTruncFlag, RS.CLGroupTruncLeft, RS.CLGroupTruncRight,\n",
    "                RS.CLChi2, RS.CLKS, RS.CLDCv, RS.CLNObs, RS.CLRunStatus],\n",
    "          ascend=[True, True, False, False, True, False, True],\n",
    "          group=[RS.CLOptimTruncFlag, RS.CLGroupTruncLeft, RS.CLGroupTruncRight]),\n",
    "     dict(key=RS.CLGblOrdDAicChi2KSDCv,\n",
    "          sort=[RS.CLParTruncLeft, RS.CLParTruncRight, RS.CLParModFitDistCuts,\n",
    "                RS.CLDeltaAic, RS.CLChi2, RS.CLKS, RS.CLDCv, RS.CLNObs, RS.CLRunStatus],\n",
    "          ascend=[True, True, True, True, False, False, True, False, True], napos='first')]\n",
    "\n",
    "results = optanlr.setupResults(ldFilSorKeySchemes=ldFilSorKeySchemes)  # Will not use predefined ones.\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert results._filterSortKeySchemes() == ldFilSorKeySchemes\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e.  _filterOnExecCode\n",
    "\n",
    "(OK, it's MCDSAnalysisResultsSet's one, but it's not any specialized in MCDS(TruncOpt)AnalysisResultsSet, so it's the same)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "results = optanlr.setupResults()\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get results table (Note: No post-computed column used here, so ... recomputation authorised, but not needed, and slower)\n",
    "dfFilSorRes = results.getData(copy=True)\n",
    "#dfFilSorRes = dfFilSorRes[dfFilSorRes[('header (head)', 'NumEchant', 'Value')] == 5].copy()  # Useful for debugging."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter params\n",
    "dupSubset = [RS.CLDensity, RS.CLDensityMin, RS.CLDensityMax]\n",
    "dDupRounds = {RS.CLDensity: 1, RS.CLDensityMin: 2, RS.CLDensityMax: 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save index before filtering\n",
    "iBefore = dfFilSorRes.index\n",
    "len(dfFilSorRes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter\n",
    "filSorSteps = ads.analyser._FilterSortSteps(filSorSchId='ExCodeTst', resultsSet=results, lang='fr')  # Steps logger\n",
    "\n",
    "RS._filterOnExecCode(dfFilSorRes, filSorSteps, results.sampleIndCol,\n",
    "                     dupSubset=dupSubset, dDupRounds=dDupRounds)\n",
    "\n",
    "# Look at steps\n",
    "filSorSteps.toList()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List filtered-out results\n",
    "sFiltered = set(iBefore) - set(dfFilSorRes.index)\n",
    "print(', '.join(str(i) for i in sFiltered))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check\n",
    "sExpected = {+0, 14, 17, 8, 9,        # sample 0: + => because of poor status elimination\n",
    "             +21, 22, 23, 27,         # sample 1: otherwise, because of non-first duplicate\n",
    "             31,                      # sample 2 ... etc.\n",
    "             +41, +39,                # sample 3\n",
    "             +56, 49, 53, 52, 57,     # sample 4\n",
    "             66, 69, 68, 65, 64}      # sample 5\n",
    "print(', '.join(str(i) for i in sExpected))\n",
    "\n",
    "assert sFiltered == sExpected, 'Oh, oh ... not what we expected'\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### f. _filterOnAicMultiQua\n",
    "\n",
    "(OK, it's MCDSAnalysisResultsSet's one, but it's not any specialized in MCDS(TruncOpt)AnalysisResultsSet, so it's the same)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "results = optanlr.setupResults()\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : we don't want it !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Get results table without re-post computation : we want post-computed columns as in source workbook !\n",
    "dfFilSorRes = results.getData(copy=True)\n",
    "#dfFilSorRes = dfFilSorRes[dfFilSorRes[('header (head)', 'NumEchant', 'Value')] == 5].copy()  # Useful for debugging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter params\n",
    "minSightRate = 92.0\n",
    "nBestAicOrd = 2\n",
    "nBestMQuaOrd = 1\n",
    "whichBestMQuaOrd = [RS.CLGrpOrdClTrChi2KSDCv, RS.CLGrpOrdClTrQuaBal3, RS.CLGrpOrdClTrQuaChi2]\n",
    "nFinalQua = 3\n",
    "whichFinalQua = RS.CLCmbQuaBal3\n",
    "ascFinalQua = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save index before filtering\n",
    "iBefore = dfFilSorRes.index\n",
    "len(dfFilSorRes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter\n",
    "filSorSteps = ads.analyser._FilterSortSteps(filSorSchId='ExAicMQuaTst', resultsSet=results, lang='fr')  # Steps logger\n",
    "\n",
    "RS._filterOnAicMultiQua(dfFilSorRes, filSorSteps, results.sampleIndCol,\n",
    "                        minSightRate=minSightRate, nBestAicOrd=nBestAicOrd,\n",
    "                        nBestMQuaOrd=nBestMQuaOrd, whichBestMQuaOrd=whichBestMQuaOrd,\n",
    "                        nFinalQua=nFinalQua, whichFinalQua=whichFinalQua, ascFinalQua=ascFinalQua)\n",
    "\n",
    "# Look at steps\n",
    "filSorSteps.toList()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List filtered-out results\n",
    "sFiltered = set(iBefore) - set(dfFilSorRes.index)\n",
    "print(', '.join(str(i) for i in sFiltered))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Auto-check filtered-out results\n",
    "# (causes => +: lower AIC, 0+: not of best multi-qua. orders, -0: poor sight rate, other: no of N best finalQua)\n",
    "sExpected =  {+5, +8, +9, 0+16, 0+17, 0+18, 14-0, 19-0, 12, # sample 0\n",
    "              +23, +24, 0+20, 0+21, +26, 30-0, 22,          # sample 1\n",
    "              +31, 0+35, 33,                                # sample 2\n",
    "              +43, 0+44, 41-0, 47-0, 39,                    # sample 3\n",
    "              +49, 0+48, 50-0, 52-0, 54-0, 57-0, 56,        # sample 4\n",
    "              +64, 0+65, 0+66, 63-0, 62}                    # sample 5\n",
    "print(', '.join(str(i) for i in sExpected))\n",
    "\n",
    "assert sFiltered == sExpected, 'Oh, oh ... not what we expected'\n",
    "\n",
    "logger.info('Success !')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = ads.logger('ads.anr', level=ads.DEBUG1)\n",
    "_ = ads.logger('ads.onr', level=ads.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### y. Non regression\n",
    "\n",
    "TODO: \n",
    "* Complete this really uncomplete and not working draft !!!\n",
    "* Is this over-simple data set really enough ? => NO !!!!\n",
    "* Isn't the current state of code below more about integration tests ?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### y.i. Load reference results\n",
    "\n",
    "(generated once through valtests.ipynb/IV. Run truncation opt-analyses ...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "refRes = optanlr.setupResults()\n",
    "\n",
    "resFileName = 'refout/ACDC2019-Naturalist-extrait-OptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "refRes.fromExcel(resFileName, postComputed=True)  # Prevent re-post-computation : this is our reference !\n",
    "\n",
    "optanlr.shutdown()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### y.ii Trigger re-post-computation on a copy\n",
    "\n",
    "(post-computations are the first thing we want to check for non regression)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ads.logger('ads.dat', level=ads.INFO, reset=True)\n",
    "ads.logger('ads.anr', level=ads.DEBUG4, reset=True)\n",
    "_ = ads.logger('ads.onr', level=ads.DEBUG4, reset=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = refRes.copy()\n",
    "res.setPostComputed(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Trigger now !\n",
    "res.dfData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### y.iii Compare re-post-computed columns to reference.\n",
    "\n",
    "TODO: Make this work ! As ... for the moment,\n",
    "* 3 results get different truncation groups\n",
    "* 29 results get different sort orders (for many or all of them)\n",
    "* out of 54 total results !"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "refRes.columns.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexCols = [('header (head)', 'NumAnlys', 'Value'),\n",
    "             ('run output', 'run folder', 'Value'),  # Needed for multiple optimisation tries kept per (same id) analysis\n",
    "             ('header (tail)', 'TrGche', 'Value'),\n",
    "             ('header (tail)', 'TrDrte', 'Value'),\n",
    "             ('header (tail)', 'NbTrchMod', 'Value'),\n",
    "             ('header (tail)', 'OptimTrunc', 'Value')]\n",
    "subsetCols=[col for col in refRes.columns if col[0] == ads.MCDSAnalysisResultsSet.CLCAutoFilSor]\n",
    "subsetCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dfComp = refRes.dfData.set_index(indexCols).sort_index()[subsetCols] \\\n",
    "         .compare(res.dfData.set_index(indexCols).sort_index()[subsetCols])\n",
    "dfComp #.to_excel('tmp/comp.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfComp[[col for col in dfComp.columns if col[2] == 'Group']].dropna(how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfComp[[col for col in dfComp.columns if col[2] == 'Order' and '(close trunc)' in col[1]]].dropna(how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfComp[[col for col in dfComp.columns if col[2] == 'Order' and '(global)' in col[1]]].dropna(how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "relDiffCols = ['Chi2 P', 'Delta AIC', 'Delta CoefVar Densité',\n",
    "               'Qual Equi 1', 'Qual Equi 2', 'Qual Equi 3', 'Qual Chi2+', 'Qual KS+', 'Qual DCv+']\n",
    "ads.DataSet.compareDataFrames(refRes.dfTransData('fr').sort_values(by='DossierExec'),\n",
    "                              res.dfTransData('fr').sort_values(by='DossierExec'),\n",
    "                              indexCols=['DossierExec'],\n",
    "                              subsetCols=relDiffCols, \n",
    "                              dropCloser=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "relDiffCols = ['TrGche', 'TrDrte', 'NbTrchMod', 'NTot Obs', 'Min Dist', 'Max Dist', 'NObs', 'NEchant', 'Effort',\n",
    "               'TxContact', 'CoefVar TxContact', 'Min TxContact', 'Max TxContact', 'DegLib TxContact', \n",
    "               'Taux Obs', 'NbTot Pars', 'AIC', 'AICc', 'BIC', 'LogProba', 'KS P', 'CvM Uw P', 'CvM Cw P',\n",
    "               'f/h(0)', 'CoefVar f/h(0)', 'Min f/h(0)', 'Max f/h(0)', 'DegLib f/h(0)',\n",
    "               'PDetec', 'CoefVar PDetec', 'Min PDetec', 'Max PDetec', 'DegLib PDetec',\n",
    "               'EDR/ESW', 'CoefVar EDR/ESW', 'Min EDR/ESW', 'Max EDR/ESW', 'DegLib EDR/ESW',\n",
    "               'DensClu', 'CoefVar DensClu', 'Min DensClu', 'Max DensClu', 'DegLib DensClu',\n",
    "               'Densité', 'Delta CoefVar Densité', 'CoefVar Densité', 'Min Densité', 'Max Densité', 'DegLib Densité',\n",
    "               'Nombre', 'CoefVar Nombre', 'Min Nombre', 'Max Nombre', 'DegLib Nombre']\n",
    "ads.DataSet.compareDataFrames(refRes.dfTransData('fr').sort_values(by='DossierExec'),\n",
    "                              res.dfTransData('fr').sort_values(by='DossierExec'),\n",
    "                              indexCols=['DossierExec'],\n",
    "                              subsetCols=relDiffCols, \n",
    "                              dropCloser=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# II. Integration tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tweak trace levels.\n",
    "ads.logger('ads.eng', level=ads.INFO, reset=True)\n",
    "if False:\n",
    "    ads.logger('ads.dat', level=ads.DEBUG, reset=True)\n",
    "    ads.logger('ads.opr', level=ads.DEBUG, reset=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. MCDSAnalyser : Run multiple analyses on real-life data (1/2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Individualised data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "countCols =  ['nMalAd10', 'nAutAd10', 'nMalAd5', 'nAutAd5']\n",
    "\n",
    "def count2AdultCat(sCounts):\n",
    "    return 'm' if 'Mal' in sCounts[sCounts > 0].index[0] else 'a'\n",
    "\n",
    "def count2DurationCat(sCounts):\n",
    "    return '5mn' if '5' in sCounts[sCounts > 0].index[0] else '10mn'\n",
    "\n",
    "fds = ads.FieldDataSet(source='refin/ACDC2019-Naturalist-extrait-ObsBrutesAvecDist.txt',\n",
    "                       importDecFields=['distMem'], countCols=countCols,\n",
    "                       addMonoCatCols={ 'Adulte': count2AdultCat, 'Durée': count2DurationCat })\n",
    "\n",
    "dfObsIndiv = fds.individualise()\n",
    "\n",
    "dfObsIndiv.drop(columns=countCols, inplace=True)\n",
    "\n",
    "dfObsIndiv.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b.Explicit analysis specs\n",
    "\n",
    "(old method: manual explicitation before run, and pass explict specs to run ;\n",
    " see 2/2 below for the new simpler and recommended method, without prior explicitation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "transectPlaceCol = 'Point'\n",
    "transectPlaceCols = [transectPlaceCol]\n",
    "passIdCol = 'Passage'\n",
    "effortCol = 'Effort'\n",
    "\n",
    "sampleDistCol = 'distMem'\n",
    "sampleDecCols=[effortCol, sampleDistCol]\n",
    "\n",
    "sampleSelCols = ['Espèce', passIdCol, 'Adulte', 'Durée']\n",
    "sampleIndCol = 'IndSamp'\n",
    "\n",
    "varIndCol = 'IndAnlys'\n",
    "anlysAbbrevCol = 'AbrevAnlys'\n",
    "\n",
    "withTruncCol = 'AvecTronc'\n",
    "\n",
    "dSurveyArea = dict(Zone='ACDC', Surface='2400')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnlysExplSpecs = ads.DSAnalyser.explicitVariantSpecs('refin/ACDC2019-Naturalist-extrait-SpecsAnalyses.xlsx', \n",
    "                                                        keep=['Echant1_impl', 'Echant2_impl', 'Modl_impl',\n",
    "                                                              'Params1_expl', 'Params2_expl'],\n",
    "                                                        varIndCol='IndAnlys',\n",
    "                                                        #convertCols={ 'Durée': int }, # float 'cause of Excel\n",
    "                                                        computedCols={anlysAbbrevCol: analysisAbbrev})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add a pass-through neutral column (for richer results)\n",
    "dfAnlysExplSpecs[withTruncCol] = dfAnlysExplSpecs[['TrGche', 'TrDrte']].apply(lambda s: s.isnull().all(), axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shorten analyses list to go faster\n",
    "if False:\n",
    "    dfAnlysExplSpecs = dfAnlysExplSpecs[(dfAnlysExplSpecs['Espèce'].isin(['Luscinia megarhynchos', 'Turdus merula']))]\n",
    "    len(dfAnlysExplSpecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnlysExplSpecs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Objet MCDSAnalyser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build the MCDSAnalyser object\n",
    "# * const effort per survey point x pass (= 1) => no need for passing transects infos (auto-generated)\n",
    "anlysr = ads.MCDSAnalyser(dfObsIndiv, effortConstVal=1, dSurveyArea=dSurveyArea,\n",
    "                          resultsHeadCols=dict(before=[varIndCol], sample=sampleSelCols, after=[withTruncCol, anlysAbbrevCol]),\n",
    "                          transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                          sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, anlysSpecCustCols=[withTruncCol],\n",
    "                          distanceUnit='Meter', areaUnit='Hectare',\n",
    "                          surveyType='Point', distanceType='Radial', clustering=False,\n",
    "                          abbrevCol=anlysAbbrevCol, anlysIndCol=varIndCol, sampleIndCol=sampleIndCol,\n",
    "                          workDir=tmpDir / 'mcds-anlr', runMethod='subprocess.run', logProgressEvery=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(anlysr.specs) == 17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysr.specs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. Check analyses specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnlysExplSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols, verdict, reasons = \\\n",
    "    anlysr.explicitParamSpecs(dfExplParamSpecs=dfAnlysExplSpecs, dropDupes=True, check=True)\n",
    "\n",
    "print(verdict, reasons, len(dfAnlysExplSpecs), userParamSpecCols, intParamSpecCols, unmUserParamSpecCols)\n",
    "\n",
    "assert len(dfAnlysExplSpecs) == 48\n",
    "assert userParamSpecCols == ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "assert intParamSpecCols == ['EstimKeyFn', 'EstimAdjustFn', 'MinDist', 'MaxDist', 'FitDistCuts']\n",
    "assert unmUserParamSpecCols == []\n",
    "assert verdict\n",
    "assert not reasons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnlysExplSpecs.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e. Run analyses\n",
    "\n",
    "(parallel mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Python 3.8, Windows 10, Core i5 8365U (4 HT cores, 1.6-4.1GHz, cache  6Mb, bus 4GT/s) + SSD 256Gb NVME + RAM 16Gb, \"optimal performances\" power scheme\n",
    "# * 2021-01 to 2021-10: 4.7s, 4.9s, 5.2s\n",
    "# Python 3.8, Windows 10, Core i7 10850H (6 HT cores, 2.7-5.1GHz, cache 12Mb, bus 8GT/s) + SSD 512Gb NVME + RAM 32Gb \"optimal performances\" power scheme\n",
    "# * 2023-11-02: 3.0s, 3.0s\n",
    "\n",
    "# Analyses\n",
    "results = anlysr.run(dfAnlysExplSpecs, threads=12)\n",
    "\n",
    "#results = anlysr.run(dfAnlysExplSpecs.iloc[:2], threads=1)  # Petit sous-ensemble pour aller vite."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysr.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "assert withTruncCol in results.dfTransData('fr').columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.toExcel(pl.Path(anlysr.workDir) / 'unintst-mcds-anlyser-results-fr.xlsx', lang='fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysr.cleanup()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. MCDSAnalyser : Run multiple analyses on real-life data (2/2)\n",
    "\n",
    "(2nd, easier and recommended version, with analysis specs checks and auto-detection of analysis parameter columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Individualised data set and analysis specs abbreviator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run [1. MCDSAnalyser : Run multiple analyses on real-life data (1/2)](#1.-MCDSAnalyser-%3A-Run-multiple-analyses-on-real-life-data-(1%2F2)) / a. and b. before"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Build MCDSAnalyser object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construction d'un MCDSAnalyser\n",
    "# * effort constant par point x passage (= 1) => pas besoin de passer les infos transects (auto-générées)\n",
    "anlysr = ads.MCDSAnalyser(dfObsIndiv, effortConstVal=1, dSurveyArea=dSurveyArea,\n",
    "                          transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                          sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                          abbrevCol=anlysAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                          anlysIndCol=varIndCol, sampleIndCol=sampleIndCol,\n",
    "                          distanceUnit='Meter', areaUnit='Hectare',\n",
    "                          surveyType='Point', distanceType='Radial', clustering=False,\n",
    "                          resultsHeadCols=dict(before=[varIndCol], sample=sampleSelCols, after=[anlysAbbrevCol]),\n",
    "                          workDir=tmpDir / 'mcds-anlr', runMethod='subprocess.run', logProgressEvery=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. Check (and explicitate) analyses specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysSpecFile = 'refin/ACDC2019-Naturalist-extrait-SpecsAnalyses.xlsx'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAnlysExplSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols, verdict, reasons = \\\n",
    "    anlysr.explicitParamSpecs(implParamSpecs=anlysSpecFile, dropDupes=True, check=True)\n",
    "\n",
    "assert len(dfAnlysExplSpecs) == 48\n",
    "assert userParamSpecCols == ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "assert intParamSpecCols == ['EstimKeyFn', 'EstimAdjustFn', 'MinDist', 'MaxDist', 'FitDistCuts']\n",
    "assert unmUserParamSpecCols == []\n",
    "assert verdict\n",
    "assert not reasons"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. Run analyses\n",
    "\n",
    "(parallel mode, and straight from implicit specs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Python 3.8, Windows 10, Core i5 8365U (4 HT cores, 1.6-4.1GHz, cache  6Mb, bus 4GT/s) + SSD 256Gb NVME + RAM 16Gb, \"optimal performances\" power scheme\n",
    "# * 2021-01 to 2021-10: 4.8s, 5.2s\n",
    "# Python 3.8, Windows 10, Core i7 10850H (6 HT cores, 2.7-5.1GHz, cache 12Mb, bus 8GT/s) + SSD 512Gb NVME + RAM 32Gb \"optimal performances\" power scheme\n",
    "# * 2023-11-02: 3.0s\n",
    "\n",
    "# Analyses (on a tout vérifié : go).\n",
    "results = anlysr.run(implParamSpecs=anlysSpecFile, threads=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysr.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.toExcel(pl.Path(anlysr.workDir) / 'unintst-mcds-anlyser-results2-fr.xlsx', lang='fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysr.cleanup()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. MCDSPreAnalyser : Run multiple pre-analyses with real-life data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Not implemented (See `valtest` notebook, chapter VIII)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. MCDSZerothOrderTruncationOptimiser : Optimise truncation params on real-life data\n",
    "\n",
    "Note: Only from explicit specs here."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### a. Jeu de données individualisées"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run [2. MCDSAnalyser : Run multiple analyses on real-life data (2/2)](#2.-MCDSAnalyser-%3A-Run-multiple-analyses-on-real-life-data-(2%2F2)) / a., b. and c. before (need for dfObsIndiv & dfAnlysExplSpecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anlysr.shutdown()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### b. Specs d'optimisation explicites"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optIndCol = 'IndOptim'\n",
    "optAbbrevCol = 'AbrevOptim'\n",
    "speAbbrevCol = 'AbrevEsp'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Left part = standard analysis params withouth truncation specs, from 4. above\n",
    "dfOptimExplSpecs = dfAnlysExplSpecs[sampleSelCols + ['FonctionClé', 'SérieAjust']].drop_duplicates().reset_index(drop=True)\n",
    "dfOptimExplSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Right part : as many as possible truncation optimisation params combinations\n",
    "dfMoreOptimCols = pd.DataFrame(dict(CritChx=[None, 'AIC']*6,\n",
    "                                    IntervConf=[None, 95, 97]*4,\n",
    "                                    TroncGche=['auto', None, 20, 'dist(5, 30)', 50.0, 'quant(3)']*2,\n",
    "                                    TroncDrte=[None, 'auto', 'dist(150, 300)', 200.0, 'tucquant(2)', 250]*2,\n",
    "                                    MethOutliers=[None, 'auto', None, None,\n",
    "                                                  None, 'quant(6)', None, None,\n",
    "                                                  None, 'tucquant(8)', None, None],\n",
    "                                    NbTrModel=[None, 9.0, 'auto', 17, 'abs(5, 10)', 'mult(0.5,5/4)']*2,\n",
    "                                    NbTrDiscr=[None, 'auto', 4, 'abs(5, 10)', 16.0, 'mult(0.5,5/4)']*2,\n",
    "                                    ExprOpt=[None, 'max(chi2)', 'min(1-chi2)', 'max(chi2)',\n",
    "                                             'max(ks)', 'max(cvmuw*cvmcw)']*2,\n",
    "                                    MoteurOpt=[None, 'zoopt', 'zoopt(mxi=20, a=racos)',\n",
    "                                               'zoopt(mxi=30, mxr=2, tv=0.5)']*3,\n",
    "                                    ParExec=[None, 'times(2)', 'times(3, b=2)']*4))\n",
    "dfMoreOptimCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concat left and right parts\n",
    "dfOptimExplSpecs = pd.concat([dfOptimExplSpecs, dfMoreOptimCols], axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add neutral and path-through columns (from specs to results) : no real use, but for testing this usefull feature\n",
    "dfOptimExplSpecs[speAbbrevCol] = dfOptimExplSpecs['Espèce'].apply(lambda s: ''.join(m[:4] for m in s.split()))\n",
    "dfOptimExplSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Artificially generate some duplicates (for testing auto-removal later :-)\n",
    "dfOptimExplSpecs = dfOptimExplSpecs.append(dfOptimExplSpecs, ignore_index=True)\n",
    "len(dfOptimExplSpecs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### c. MCDSZerothOrderTruncationOptimiser object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Colonnes de dfOptimExplSpecs donnant les paramètres d'analyse / optimisation\n",
    "optimParamsSpecsCols  = ['FonctionClé', 'SérieAjust', 'CritChx', 'IntervConf',\n",
    "                         'TroncGche', 'TroncDrte', 'MethOutliers', 'NbTrModel', 'NbTrDiscr',\n",
    "                         'ExprOpt', 'MoteurOpt', 'ParExec']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "zoptr = ads.MCDSZerothOrderTruncationOptimiser \\\n",
    "                (dfObsIndiv, effortConstVal=1, dSurveyArea=dSurveyArea, \n",
    "                 transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                 sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                 anlysSpecCustCols=[speAbbrevCol], abbrevCol=optAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                 anlysIndCol=optIndCol, sampleIndCol=sampleIndCol,\n",
    "                 distanceUnit='Meter', areaUnit='Hectare',\n",
    "                 surveyType='Point', distanceType='Radial', clustering=False,\n",
    "                 resultsHeadCols=dict(before=[optIndCol], sample=sampleSelCols, after=optimParamsSpecsCols + [speAbbrevCol]),\n",
    "                 workDir=tmpDir / 'mcds-optr', runMethod='subprocess.run', runTimeOut=None,\n",
    "                 logData=False, logProgressEvery=1, backupEvery=5,\n",
    "                 defEstimKeyFn='HAZ', defEstimAdjustFn='POLY', defEstimCriterion='AIC', defCVInterval=93,\n",
    "                 defExpr2Optimise='1-ks', defMinimiseExpr=True,\n",
    "                 defOutliersMethod='quant', defOutliersQuantCutPct=5.5,\n",
    "                 defFitDistCutsFctr=dict(min=1/2, max=4/3), defDiscrDistCutsFctr=dict(min=1/2, max=1.2),\n",
    "                 defSubmitTimes=4, defSubmitOnlyBest=1,\n",
    "                 defCoreMaxIters=45, defCoreTermExprValue=0.2, defCoreMaxRetries=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### d. Vérification des specs d'optimisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfOptimExplSpecs, userParamSpecCols, intParamSpecCols, unmUserParamSpecCols, verdict, reasons = \\\n",
    "    zoptr.explicitParamSpecs(dfExplParamSpecs=dfOptimExplSpecs, dropDupes=True, check=True)\n",
    "\n",
    "assert len(dfOptimExplSpecs) == 12\n",
    "assert userParamSpecCols == optimParamsSpecsCols\n",
    "assert intParamSpecCols == ['EstimKeyFn', 'EstimAdjustFn', 'EstimCriterion', 'CvInterval',\n",
    "                            'MinDist', 'MaxDist', 'OutliersMethod', 'FitDistCuts', 'DiscrDistCuts',\n",
    "                            'Expr2Optimise', 'OptimisationCore', 'SubmitParams']\n",
    "assert unmUserParamSpecCols == []\n",
    "assert verdict\n",
    "assert not reasons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dfOptimExplSpecs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### e. Exécution des optimisations\n",
    "\n",
    "(en parallèle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Python 3.8, Windows 10, Core i5 8365U (4 HT cores, 1.6-4.1GHz, cache  6Mb, bus 4GT/s) + SSD 256Gb NVME + RAM 16Gb, \"optimal performances\" power scheme\n",
    "# * 2021-01: 12 optimisations, 1430 analyses, 12 threads : subprocess = 3mn13, system = 2mn35, 1mn54?\n",
    "# * 2021-10-02: idem : system 2mn15\n",
    "# Python 3.8, Windows 10, Core i7 10850H (6 HT cores, 2.7-5.1GHz, cache 12Mb, bus 8GT/s) + SSD 512Gb NVME + RAM 32Gb \"optimal performances\" power scheme\n",
    "# * 2023-11-02: 12 optimisations, 1434 analyses, 12 threads : system = 1mn44s, 1mn46s, subprocess = 1mn47s\n",
    "#               (N.B. non saturant = max 6 MCDS en // observés => augmenter threads . jeu top petit ?)\n",
    "\n",
    "#results = zoptr.run(dfOptimExplSpecs.iloc[:3], threads=3)  # Small subset for quicker run.\n",
    "#results = zoptr.run(dfOptimExplSpecs.iloc[:6], threads=6)  # Half subset for quicker run, with 1 recovery file.\n",
    "results = zoptr.run(dfOptimExplSpecs, threads=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "zoptr.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert speAbbrevCol in results.dfTransData('fr').columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.dfTransData('fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.toExcel(pl.Path(zoptr.workDir) / 'unintst-mcds-optimiser-results-fr.xlsx', lang='fr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### f. Recovery : Run again optimisations, but from the last backup\n",
    "\n",
    "(use case: crash, or mandatory/auto reboot of computer in the middle of a long optimisation run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# i. Quickly check content of the recovery file\n",
    "optResBkpPath = pl.Path(zoptr.workDir / 'optr-resbak-0.pickle.xz')\n",
    "with lzma.open(optResBkpPath, 'rb') as file:\n",
    "    dfData, specs = pickle.load(file)\n",
    "assert len(dfData) == 17\n",
    "exptdCols = [optIndCol] + sampleSelCols + optimParamsSpecsCols + [speAbbrevCol]\n",
    "exptdCols += ['OptAbbrev', 'KeyFn', 'AdjSer', 'EstCrit', 'CVInt', 'OptCrit',\n",
    "              'MinDist', 'MaxDist', 'FitDistCuts', 'DiscrDistCuts',\n",
    "              'SetupStatus', 'SubmitStatus', 'NFunEvals', 'MeanFunElapd',\n",
    "              '1-ks', 'maxDist', 'discrDistCuts', 'chi2', 'minDist',\n",
    "              'fitDistCuts', 'ks', '1-chi2', 'cvmuw*cvmcw']\n",
    "assert dfData.columns.tolist() == exptdCols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ii. Create the optimiser object : have to be a clone of the one whose execution was backed up\n",
    "zoptr = ads.MCDSZerothOrderTruncationOptimiser \\\n",
    "                (dfObsIndiv, effortConstVal=1, dSurveyArea=dSurveyArea, \n",
    "                 transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                 sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols, sampleDistCol=sampleDistCol,\n",
    "                 anlysSpecCustCols=[speAbbrevCol], abbrevCol=optAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                 anlysIndCol=optIndCol, sampleIndCol=sampleIndCol,\n",
    "                 distanceUnit='Meter', areaUnit='Hectare',\n",
    "                 surveyType='Point', distanceType='Radial', clustering=False,\n",
    "                 resultsHeadCols=dict(before=[optIndCol], sample=sampleSelCols, after=optimParamsSpecsCols + [speAbbrevCol]),\n",
    "                 workDir=tmpDir / 'mcds-optr', logProgressEvery=1,\n",
    "                 defEstimKeyFn='HAZ', defEstimAdjustFn='POLY', defEstimCriterion='AIC', defCVInterval=93,\n",
    "                 defExpr2Optimise='1-ks', defMinimiseExpr=True,\n",
    "                 defOutliersMethod='quant', defOutliersQuantCutPct=5.5,\n",
    "                 defFitDistCutsFctr=dict(min=1/2, max=4/3), defDiscrDistCutsFctr=dict(min=1/2, max=1.2),\n",
    "                 defSubmitTimes=4, defSubmitOnlyBest=1,\n",
    "                 defCoreMaxIters=45, defCoreTermExprValue=0.2, defCoreMaxRetries=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# iii.Run optimisation with recovery results ... using exact same optim. specs (MANDATORY)\n",
    "#results2 = zoptr.run(dfOptimExplSpecs.iloc[:3], recover=True, threads=3)  # Small subset for quicker run.\n",
    "#results2 = zoptr.run(dfOptimExplSpecs.iloc[:3], recover=True, threads=3)  # Half subset for quicker run, with 1 recovery file.\n",
    "results2 = zoptr.run(dfOptimExplSpecs, recover=True, threads=12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "zoptr.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results2.dfData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results2.toExcel(pl.Path(zoptr.workDir) / 'unintst-mcds-optimiser-results2-fr.xlsx', lang='fr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iv. Quickly check results\n",
    "assert len(results2) == 20 # Given the ParExec column of dfOptimExplSpecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# v. Check equality of 1st 17 results in `results` and `results2`, + added num of results\n",
    "# (20 results at the end, 1st 17 ones in backup file, so only reloaded => only 3 left to be recomputed)\n",
    "dfComp = results.dfTransData(lang='fr').compare(results2.dfTransData(lang='fr'))\n",
    "logger.info(f'First run vs recovery compared results: n={len(dfComp)} =>\\n' + dfComp.to_string())\n",
    "\n",
    "assert len(dfComp) <= 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. MCDSTruncationOptAnalyser : Run multiple analyses with optimised truncation params, on real-life data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not implemented here (but see [valtests.ipynb](valtests.ipynb#IV.-Run-truncation-opt-analyses-with-same-real-life-field-data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dev : Optimise MCDSAnalyser._postComputeQualityIndicators"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Prepare stuff for creating MCDSAnalysisResultsSet objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source / Results data\n",
    "transectPlaceCols = ['Point']\n",
    "passIdCol = 'Passage'\n",
    "effortCol = 'Effort'\n",
    "\n",
    "sampleDistCol = 'Distance'\n",
    "sampleDecCols = [effortCol, sampleDistCol]\n",
    "\n",
    "sampleNumCol = 'NumEchant'\n",
    "sampleSelCols = ['Espèce', passIdCol, 'Adulte', 'Durée']\n",
    "\n",
    "sampleAbbrevCol = 'AbrevEchant'\n",
    "\n",
    "dSurveyArea = dict(Zone='ACDC', Surface='2400')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# General DS analysis parameters\n",
    "varIndCol = 'NumAnlys'\n",
    "anlysAbbrevCol = 'AbrevAnlys'\n",
    "anlysParamCols = ['FonctionClé', 'SérieAjust', 'TrGche', 'TrDrte', 'NbTrchMod']\n",
    "\n",
    "distanceUnit = 'Meter'\n",
    "areaUnit = 'Hectare'\n",
    "surveyType = 'Point'\n",
    "distanceType = 'Radial'\n",
    "clustering = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results post-computation parameters\n",
    "ldTruncIntrvSpecs = [dict(col='left', minDist=5.0, maxLen=5.0), dict(col='right', minDist=25.0, maxLen=25.0)]\n",
    "truncIntrvEpsilon = 1e-6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load individualised observations and actual transects\n",
    "indivObsFile = 'refin/ACDC2019-Naturalist-extrait-ObsIndiv.ods'\n",
    "\n",
    "dfObsIndiv = ads.DataSet(indivObsFile, sheet='DonnéesIndiv').dfData\n",
    "\n",
    "dfTransects = ads.DataSet(indivObsFile, sheet='Inventaires').dfData\n",
    "\n",
    "dict(indivObs=len(dfObsIndiv), transects=len(dfTransects))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# What's better to create an MCDSAnalysisResultsSet objets than a MCDSAnalyser instance ?\n",
    "anlr = \\\n",
    "    ads.MCDSAnalyser(dfObsIndiv, dfTransects=dfTransects, dSurveyArea=dSurveyArea, \n",
    "                     transectPlaceCols=transectPlaceCols, passIdCol=passIdCol, effortCol=effortCol,\n",
    "                     sampleSelCols=sampleSelCols, sampleDecCols=sampleDecCols,\n",
    "                     abbrevCol=anlysAbbrevCol, abbrevBuilder=analysisAbbrev,\n",
    "                     anlysIndCol=varIndCol, sampleIndCol=sampleNumCol,\n",
    "                     distanceUnit=distanceUnit, areaUnit=areaUnit,\n",
    "                     surveyType=surveyType, distanceType=distanceType, clustering=clustering,\n",
    "                     resultsHeadCols=dict(before=[varIndCol, sampleNumCol], sample=sampleSelCols,\n",
    "                                          after=anlysParamCols + [anlysAbbrevCol]),\n",
    "                     ldTruncIntrvSpecs=ldTruncIntrvSpecs, truncIntrvEpsilon=truncIntrvEpsilon)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. _postComputeQualityIndicators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load results to play with ...\n",
    "# Note: Okay, it's actually an MCDSTruncOptAnalysisResultsSet file ... but we'll ignore the extra columns, promised :-)\n",
    "resFileName = 'refin/ACDC2019-Naturalist-UnitestOptResultats.ods'\n",
    "print('Loading results from {} ...'.format(resFileName))\n",
    "\n",
    "results = anlr.setupResults()\n",
    "\n",
    "results.fromOpenDoc(resFileName, postComputed=True)  # Prevent re-post-computation : not a problem here, but longer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Results(object):\n",
    "    \n",
    "    def __init__(self, dfData):\n",
    "        \n",
    "        self._dfData = dfData\n",
    "    \n",
    "    # Post computations : Quality indicators.\n",
    "    DNormKeyFn = dict(HNORMAL=1.0, UNIFORM=0.9, HAZARD=0.6, NEXPON=0.1)\n",
    "    # DNormKeyFn = dict(HNORMAL=1.00, UNIFORM=0.75, HAZARD=0.5, NEXPON=0.1)  # Not better\n",
    "\n",
    "    @classmethod\n",
    "    def _normNTotPars(cls, sRes, a=0.2, b=0.6, c=2):  # , d=1):\n",
    "        return 1 / (a * max(c, sRes[cls.CLNTotPars]) + b)  # Mieux: a=0.2, b=0.6, c=2 / a=0.2, b=0.8, c=1\n",
    "        # return 1 / (a * max(c, sRes[cls.CLNTotPars])**d + b)  # Idem (d=1)\n",
    "\n",
    "    @classmethod\n",
    "    def _combinedQualityBalanced1(cls, sRes):  # The one used for ACDC 2019 filtering & sorting in jan/feb 2021\n",
    "        TODOOOOOOOOOOOOOOOOOOOOOOOOOO like _combinedQualityAll\n",
    "        return (sRes[[cls.CLChi2, cls.CLKS, cls.CLCvMUw, cls.CLCvMCw]].prod()\n",
    "                * cls._normNObs(sRes) * cls._normNTotPars(sRes, a=0.2, b=0.6)\n",
    "                * cls._normCVDens(sRes, a=12, b=2)) ** (1.0/7)\n",
    "\n",
    "    CLsQuaIndicSources = [CLKeyFn, CLNAdjPars, CLNTotPars, CLNObs, CLNTotObs, CLChi2, CLKS, CLCvMUw, CLCvMCw, CLDCv]\n",
    "    \n",
    "    CIKeyFn = CLsQuaIndicSources.index(CLKeyFn)\n",
    "    CINAdjPars = CLsQuaIndicSources.index(CLNAdjPars)\n",
    "    CINTotPars = CLsQuaIndicSources.index(CLNTotPars)\n",
    "    CINObs = CLsQuaIndicSources.index(CLNObs)\n",
    "    CINTotObs = CLsQuaIndicSources.index(CLNTotObs)\n",
    "    CIChi2 = CLsQuaIndicSources.index(CLChi2)\n",
    "    CIKS = CLsQuaIndicSources.index(CLKS)\n",
    "    CICvMUw = CLsQuaIndicSources.index(CLCvMUw)\n",
    "    CICvMCw = CLsQuaIndicSources.index(CLCvMCw)\n",
    "    CIDCv = CLsQuaIndicSources.index(CLDCv)\n",
    "    \n",
    "    @classmethod\n",
    "    def _combinedQualityAll(cls, aRes):\n",
    "        \n",
    "        #\n",
    "        chi2 = aRes[cls.CIChi2]\n",
    "        ks = aRes[cls.CIKS]\n",
    "        chi2 = aRes[cls.CIChi2]\n",
    "        chi2KsCvMs = aRes[cls.CIChi2:cls.CICvMCw + 1].prod()\n",
    "        normNObs = aRes[cls.CINObs] / aRes[cls.CINTotObs]\n",
    "        \n",
    "        # October 2021\n",
    "        normKeyFn = cls.DNormKeyFn.get(aRes[cls.CIKeyFn], 0.0)\n",
    "\n",
    "        # A more devaluating version for NAdjPars, CVDens, also using KeyFn\n",
    "        normNAdjPars = math.exp(-0.15 * aRes[cls.CINAdjPars] * aRes[cls.CINAdjPars])\n",
    "        normCVDens = math.exp(-20 * aRes[cls.CIDCv] * aRes[cls.CIDCv])\n",
    "        prodAll8NormSrcIndics = chi2KsCvMs * normNObs * normNAdjPars * normCVDens * normKeyFn\n",
    "        quaBal2 = prodAll8NormSrcIndics ** 0.125\n",
    "\n",
    "        # An even more devaluating version for NAdjPars, CVDens, also using \n",
    "        normNAdjPars = math.exp(-0.17 * aRes[cls.CINAdjPars] * aRes[cls.CINAdjPars])\n",
    "        normCVDens = math.exp(-63 * aRes[cls.CIDCv] ** 2.8)\n",
    "        prodAll8NormSrcIndics = chi2KsCvMs * normNObs * normNAdjPars * normCVDens * normKeyFn\n",
    "        quaBal3 = prodAll8NormSrcIndics ** 0.125\n",
    "\n",
    "        # Follow _combinedQualityBalanced3 update (were based on _combinedQualityBalanced1)\n",
    "        moreChi2 = (prodAll8NormSrcIndics * chi2) ** (1.0/9)\n",
    "        moreKS = (prodAll8NormSrcIndics * ks) ** (1.0/9)\n",
    "        moreDCv = (prodAll8NormSrcIndics * normCVDens) ** (1.0/9)\n",
    "        \n",
    "        return quaBal2, quaBal3, moreChi2, moreKS, moreDCv  # Must be same order as in CLsQuaIndics !\n",
    "\n",
    "    CLsQuaIndics = [CLCmbQuaBal2, CLCmbQuaBal3, CLCmbQuaChi2, CLCmbQuaKS, CLCmbQuaDCv]\n",
    "\n",
    "    def _postComputeQualityIndicators(self):\n",
    "        \n",
    "        cls = self\n",
    "\n",
    "        logger.debug('Post-computing Quality Indicators')\n",
    "\n",
    "        #self._dfData.apply(self._normNObs, axis='columns')\n",
    "        self._dfData[cls.CLSightRate] = 100 * self._dfData[cls.CLNObs] / self._dfData[cls.CLNTotObs]  # [0,1] => %\n",
    "\n",
    "        # Prepare data for computations\n",
    "        logger.debug1('* Pre-processing source data')\n",
    "\n",
    "        # a. extract the useful columns, after adding them if not present\n",
    "        #    (NaN value, except for CLKeyFn, that MUST be there anyway)\n",
    "        for miCol in cls.CLsQuaIndicSources:\n",
    "            if miCol not in self._dfData.columns and miCol != cls.CLKeyFn:\n",
    "                self._dfData[miCol] = np.nan\n",
    "        dfCompData = self._dfData[cls.CLsQuaIndicSources].copy()\n",
    "\n",
    "        # b. historical bal qua 1\n",
    "        logger.debug1('* Balanced quality 1')\n",
    "        self._dfData[cls.CLCmbQuaBal1] = dfCompData.apply(cls._combinedQualityBalanced1, axis='columns')\n",
    "\n",
    "        # c. newer quality indicators\n",
    "        #    (NaN value MUST kill down these indicators to compute => we have to enforce this)\n",
    "        dfCompData.fillna({cls.CLNObs: cls.KilrNObs,\n",
    "                           cls.CLChi2: cls.KilrStaTest, cls.CLKS: cls.KilrStaTest,\n",
    "                           cls.CLCvMUw: cls.KilrStaTest, cls.CLCvMCw: cls.KilrStaTest,\n",
    "                           cls.CLDCv: cls.KilrDensCv,  # Usually considered good under 0.3\n",
    "                           cls.CLNTotObs: cls.KilrNTotObs,  # Should slap down _normObs whatever NObs\n",
    "                           cls.CLNAdjPars: cls.KilrNPars,  # Should slap down _normNAdjPars whatever NObs\n",
    "                           cls.CLNTotPars: cls.KilrNPars},\n",
    "                          inplace=True)\n",
    "\n",
    "        #logger.debug1('* Balanced quality 2')\n",
    "        #self._dfData[cls.CLCmbQuaBal2] = dfCompData.apply(cls._combinedQualityBalanced2, axis='columns')\n",
    "        #logger.debug1('* Balanced quality 3')\n",
    "        #self._dfData[cls.CLCmbQuaBal3] = dfCompData.apply(cls._combinedQualityBalanced3, axis='columns')\n",
    "        #logger.debug1('* Balanced quality Chi2+')\n",
    "        #self._dfData[cls.CLCmbQuaChi2] = dfCompData.apply(cls._combinedQualityMoreChi2, axis='columns')\n",
    "        #logger.debug1('* Balanced quality KS+')\n",
    "        #self._dfData[cls.CLCmbQuaKS] = dfCompData.apply(cls._combinedQualityMoreKS, axis='columns')\n",
    "        #logger.debug1('* Balanced quality DCv+')\n",
    "        #self._dfData[cls.CLCmbQuaDCv] = dfCompData.apply(cls._combinedQualityMoreDCv, axis='columns')\n",
    "\n",
    "        logger.debug1('* Balanced quality 2, 3, Chi2+, KS+, DCv+')\n",
    "        self._dfData.drop(columns=cls.CLsQuaIndics, inplace=True, error='ignore')\n",
    "        aQuaIndics = dfCompData.apply(_combinedQualityAll, axis='columns', raw=True).values\n",
    "        self._dfData = self._dfData.join(pd.DataFrame(aQuaIndics, index=self._dfData.index,\n",
    "                                                      columns=pd.MultiIndex.from_tuples(cls.CLsQuaIndics)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sandbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8",
   "language": "python",
   "name": "py38"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}