Fix and update flatten_timepoint_specific_output_overrides

Closes #247

Was creating wrong observables before

petab/core.py

@@ -8,7 +8,6 @@
 import pandas as pd
 import sympy as sp
 
-from . import sbml
 from .C import *  # noqa: F403
 
 
@@ -148,7 +147,8 @@ def flatten_timepoint_specific_output_overrides(
     replace those by replicating the respective observable.
 
     This is a helper function for some tools which may not support such
-    timepoint-specific mappings. The measurement table is modified in place.
+    timepoint-specific mappings. The observable table and measurement table
+    are modified in place.
 
     Arguments:
         petab_problem:
@@ -169,6 +169,8 @@ def flatten_timepoint_specific_output_overrides(
     # and simulationConditionId
     df_unique_values = df.drop_duplicates()
 
+    # replaced observables: new ID => old ID
+    replacements = dict()
     # Loop over each unique combination
     for nrow in range(len(df_unique_values.index)):
         df = petab_problem.measurement_df.loc[
@@ -206,6 +208,8 @@ def flatten_timepoint_specific_output_overrides(
                 ) == 0).any():
                     tmp = tmp_ + counter*"_" + str(i_noise + i_sc + 1)
                     counter += 1
+                if not tmp_.empty and not tmp_.empty:
+                    replacements[tmp.values[0]] = tmp_.values[0]
                 df.loc[idxs == 0, OBSERVABLE_ID] = tmp
                 # Append the result in a new df
                 df_new = df_new.append(df.loc[idxs == 0])
@@ -216,32 +220,20 @@ def flatten_timepoint_specific_output_overrides(
     # Update/Redefine measurement df with replicate-specific observables
     petab_problem.measurement_df = df_new
 
-    # Get list of already existing unique observable names
-    unique_observables = df[OBSERVABLE_ID].unique()
-
-    # Remove already existing observables from the sbml model
-    for obs in unique_observables:
-        petab_problem.sbml_model.removeRuleByVariable("observable_" + obs)
-        petab_problem.sbml_model.removeSpecies(obs)
-        petab_problem.sbml_model.removeParameter(
-            'observable_' + obs)
-
-    # Redefine with replicate-specific observables in the sbml model
-    for replicate_id in petab_problem.measurement_df[OBSERVABLE_ID].unique():
-        sbml.add_global_parameter(
-            sbml_model=petab_problem.sbml_model,
-            parameter_id='observableParameter1_' + replicate_id)
-        sbml.add_global_parameter(
-            sbml_model=petab_problem.sbml_model,
-            parameter_id='noiseParameter1_' + replicate_id)
-        sbml.add_model_output(
-            sbml_model=petab_problem.sbml_model,
-            observable_id=replicate_id,
-            formula='observableParameter1_' + replicate_id)
-        sbml.add_model_output_sigma(
-            sbml_model=petab_problem.sbml_model,
-            observable_id=replicate_id,
-            formula='noiseParameter1_' + replicate_id)
+    # Update observables table
+    for replacement, replacee in replacements.items():
+        new_obs = petab_problem.observable_df.loc[replacee].copy()
+        new_obs.name = replacement
+        new_obs[OBSERVABLE_FORMULA] = new_obs[OBSERVABLE_FORMULA].replace(
+            replacee, replacement)
+        new_obs[NOISE_FORMULA] = new_obs[NOISE_FORMULA].replace(
+            replacee, replacement)
+        petab_problem.observable_df = petab_problem.observable_df.append(
+            new_obs
+        )
+
+    petab_problem.observable_df.drop(index=set(replacements.values()),
+                                     inplace=True)
 
 
 def concat_tables(

petab/lint.py

@@ -470,8 +470,8 @@ def measurement_table_has_timepoint_specific_mappings(
     grouped_df2 = grouped_df.groupby(grouping_cols).size().reset_index()
 
     if len(grouped_df.index) != len(grouped_df2.index):
-        logger.warning(
-            "Measurement table has timepoint specific mappings:\n{grouped_df}")
+        logger.warning("Measurement table has timepoint-specific "
+                       f"mappings:\n{grouped_df}")
         return True
     return False
 

tests/test_petab.py

@@ -275,13 +275,26 @@ def test_create_parameter_df(condition_df_2_conditions):
     assert parameter_df.loc['p0', NOMINAL_VALUE] == 3.0
 
 
-def test_flatten_timepoint_specific_output_overrides(minimal_sbml_model):
-    document, model = minimal_sbml_model
-    petab.sbml.add_global_parameter(
-        sbml_model=model, parameter_id='observableParameter1_obs1')
-    petab.sbml.add_model_output_with_sigma(
-        sbml_model=model, observable_id='obs1',
-        observable_formula='observableParameter1_obs1')
+def test_flatten_timepoint_specific_output_overrides():
+    observable_df = pd.DataFrame(data={
+        OBSERVABLE_ID: ['obs1'],
+        OBSERVABLE_FORMULA: [
+            'observableParameter1_obs1 + observableParameter2_obs1'],
+        NOISE_FORMULA: ['noiseParameter1_obs1']
+    })
+    observable_df.set_index(OBSERVABLE_ID, inplace=True)
+
+    observable_df_expected = pd.DataFrame(data={
+        OBSERVABLE_ID: ['obs1_1', 'obs1_2', 'obs1_3'],
+        OBSERVABLE_FORMULA: [
+            'observableParameter1_obs1_1 + observableParameter2_obs1_1',
+            'observableParameter1_obs1_2 + observableParameter2_obs1_2',
+            'observableParameter1_obs1_3 + observableParameter2_obs1_3'],
+        NOISE_FORMULA: ['noiseParameter1_obs1_1',
+                        'noiseParameter1_obs1_2',
+                        'noiseParameter1_obs1_3']
+    })
+    observable_df_expected.set_index(OBSERVABLE_ID, inplace=True)
 
     # Measurement table with timepoint-specific overrides
     measurement_df = pd.DataFrame(data={
@@ -296,8 +309,8 @@ def test_flatten_timepoint_specific_output_overrides(minimal_sbml_model):
         MEASUREMENT:
             [np.nan] * 4,
         OBSERVABLE_PARAMETERS:
-            ['obsParOverride1', 'obsParOverride2',
-             'obsParOverride2', 'obsParOverride2'],
+            ['obsParOverride1;1.0', 'obsParOverride2;1.0',
+             'obsParOverride2;1.0', 'obsParOverride2;1.0'],
         NOISE_PARAMETERS:
             ['noiseParOverride1', 'noiseParOverride1',
              'noiseParOverride2', 'noiseParOverride2']
@@ -315,15 +328,15 @@ def test_flatten_timepoint_specific_output_overrides(minimal_sbml_model):
         MEASUREMENT:
             [np.nan] * 4,
         OBSERVABLE_PARAMETERS:
-            ['obsParOverride1', 'obsParOverride2',
-             'obsParOverride2', 'obsParOverride2'],
+            ['obsParOverride1;1.0', 'obsParOverride2;1.0',
+             'obsParOverride2;1.0', 'obsParOverride2;1.0'],
         NOISE_PARAMETERS:
             ['noiseParOverride1', 'noiseParOverride1',
              'noiseParOverride2', 'noiseParOverride2']
     })
 
-    problem = petab.Problem(sbml_model=model,
-                            measurement_df=measurement_df)
+    problem = petab.Problem(measurement_df=measurement_df,
+                            observable_df=observable_df)
 
     assert petab.lint_problem(problem) is False
 
@@ -337,6 +350,7 @@ def test_flatten_timepoint_specific_output_overrides(minimal_sbml_model):
     assert petab.lint.measurement_table_has_timepoint_specific_mappings(
         problem.measurement_df) is False
 
+    assert problem.observable_df.equals(observable_df_expected) is True
     assert problem.measurement_df.equals(measurement_df_expected) is True
 
     assert petab.lint_problem(problem) is False