Skip to content

Transition to tlo logging

Jump to bottom
Stefan Piatek edited this page Feb 11, 2020 · 5 revisions

Process

  • Pull master into your development branch
  • For any merge conflicts, please review them the assumption that accept theirs is right (this means accepting the changes from master). See Cookbook#logging if merge conflicts appear that are unclear.
  • Update your files.(changes to lines are highlighted with # <--), in old usage these lines are deleted and in new usage new lines are added or updated. The list of the steps to change are in the new usage subheader for each file type.

Tests

  • For tests in development, please remove the disable_logging method from your test files if you have one and make sure to change import logging to from tlo import logging.
  • All existing tests have now been changed over to using new style so nothing needs to be changed.
  • You may want to look at Cookbook#logging when developing new tests if you want to test the logging output dataframe or ignore logging of other modules.

Disease Modules

Old usage

import logging  # <--
import math
from pathlib import Path

import numpy as np
import pandas as pd

from tlo import DateOffset, Module, Parameter, Property, Types
# ...

New usage

  • Remove import logging
  • add logging to the line with from tlo import
    • This should be after Capital imports, and in alphabetical order
  • the rest of your interaction with logging is the same!
import math
from pathlib import Path

import numpy as np
import pandas as pd

from tlo import DateOffset, Module, Parameter, Property, Types, logging  # <--
# ...

Analysis scripts, all logging enabled

Old usage

import datetime
import logging  # <--
import os
from pathlib import Path

from tlo import Date, Simulation
from tlo.analysis.utils import parse_log_file
from tlo.methods import contraception, demography, enhanced_lifestyle

# Where will outputs go - by default, wherever this script is run
outputpath = Path("./outputs")  # folder for convenience of storing outputs


# date-stamp to label log files and any other outputs
datestamp = datetime.date.today().strftime("__%Y_%m_%d")

# The resource file for demography module
# assume Python console is started in the top-leve TLOModel directory
resourcefilepath = Path("./resources")

# %% Run the Simulation

start_date = Date(2010, 1, 1)
end_date = Date(2050, 1, 1)
popsize = 1000

# add file handler for the purpose of logging
sim = Simulation(start_date=start_date)

# this block of code is to capture the outputs to file
logfile = outputpath / ('LogFile' + datestamp + '.log')    # <--

if os.path.exists(logfile):    # <--
    os.remove(logfile)    # <--
fh = logging.FileHandler(logfile)    # <--
fr = logging.Formatter("%(levelname)s|%(name)s|%(message)s")    # <--
fh.setFormatter(fr)    # <--
logging.getLogger().addHandler(fh)    # <--

# run the simulation
sim.register(demography.Demography(resourcefilepath=resourcefilepath))
sim.register(enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath))
sim.register(contraception.Contraception(resourcefilepath=resourcefilepath))

sim.seed_rngs(1)
sim.make_initial_population(n=popsize)
sim.simulate(end_date=end_date)

# this will make sure that the logging file is complete  # <--
fh.flush()  # <--

New usage

  • remove import logging
  • add logging to the line with from tlo import
    • This should be after Capital imports, and in alphabetical order
  • remove lines building the logfile path (this is now returned by sim.configure_logging)
  • remove lines setting the file handler (this is now done in sim.configure_logging)
  • after all modules have been registered with the simulation
    • call sim.configure_logging to set the logfile name, assigning the output of this to keep the full logfile path.
  • remove your fh.flush() and fh.close(), this is already done in the simulation now
import datetime
import os
from pathlib import Path

from tlo import Date, Simulation, logging  # <--
from tlo.analysis.utils import parse_log_file
from tlo.methods import contraception, demography, enhanced_lifestyle

# Where will outputs go - by default, wherever this script is run
outputpath = Path("./outputs")  # folder for convenience of storing outputs


# date-stamp to label log files and any other outputs
datestamp = datetime.date.today().strftime("__%Y_%m_%d")

# The resource file for demography module
# assume Python console is started in the top-leve TLOModel directory
resourcefilepath = Path("./resources")

# %% Run the Simulation

start_date = Date(2010, 1, 1)
end_date = Date(2050, 1, 1)
popsize = 1000

sim = Simulation(start_date=start_date)

# run the simulation
sim.register(demography.Demography(resourcefilepath=resourcefilepath))
sim.register(enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath))
sim.register(contraception.Contraception(resourcefilepath=resourcefilepath))

logfile = sim.configure_logging(filename="LogFile")  # <--

sim.seed_rngs(1)
sim.make_initial_population(n=popsize)
sim.simulate(end_date=end_date)

# %% read the results
output = parse_log_file(logfile)

Analysis scripts, only logging specific modules

Old usage

import datetime
import logging  # <--
import os
from pathlib import Path

from tlo import Date, Simulation
from tlo.analysis.utils import parse_log_file
from tlo.methods import (
    demography,
    enhanced_lifestyle,
    healthburden,
    healthsystem,
    hiv,
    male_circumcision,
    tb,
)

# Where will outputs go
outputpath = Path("./outputs")  # folder for convenience of storing outputs

# date-stamp to label log files and any other outputs
datestamp = datetime.date.today().strftime("__%Y_%m_%d")

# The resource files
resourcefilepath = Path("./resources")

start_date = Date(2010, 1, 1)
end_date = Date(2015, 1, 1)
popsize = 5000

# Establish the simulation object
sim = Simulation(start_date=start_date)

# Establish the logger  # <--
logfile = outputpath / ("LogFile" + datestamp + ".log") # <--

if os.path.exists(logfile):  # <--
    os.remove(logfile)  # <--
fh = logging.FileHandler(logfile)  # <--
fr = logging.Formatter("%(levelname)s|%(name)s|%(message)s")  # <--
fh.setFormatter(fr)  # <--
logging.getLogger().addHandler(fh)  # <--

# ----- Control over the types of intervention that can occur -----
# Make a list that contains the treatment_id that will be allowed. Empty list means nothing allowed.
# '*' means everything. It will allow any treatment_id that begins with a stub (e.g. Mockitis*)
service_availability = ["*"]

logging.getLogger("tlo.methods.demography").setLevel(logging.WARNING)  # <--
logging.getLogger("tlo.methods.lifestyle").setLevel(logging.WARNING)  # <--
logging.getLogger("tlo.methods.healthburden").setLevel(logging.WARNING)  # <--
logging.getLogger("tlo.methods.hiv").setLevel(logging.INFO)  # <--
logging.getLogger("tlo.methods.tb").setLevel(logging.INFO)  # <--
logging.getLogger("tlo.methods.male_circumcision").setLevel(logging.INFO)  # <--

# Register the appropriate modules
sim.register(demography.Demography(resourcefilepath=resourcefilepath))
sim.register(healthsystem.HealthSystem(resourcefilepath=resourcefilepath))
sim.register(healthburden.HealthBurden(resourcefilepath=resourcefilepath))
sim.register(enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath))
sim.register(hiv.hiv(resourcefilepath=resourcefilepath))
sim.register(tb.tb(resourcefilepath=resourcefilepath))
sim.register(male_circumcision.male_circumcision(resourcefilepath=resourcefilepath))

# Run the simulation and flush the logger
sim.seed_rngs(0)
sim.make_initial_population(n=popsize)
sim.simulate(end_date=end_date)
fh.flush()  # <--

New usage

  • remove import logging
  • add logging to the line with from tlo import
    • This should be after Capital imports, and in alphabetical order
  • remove lines building the logfile path (this is now returned by sim.configure_logging)
  • remove lines setting the file handler (this is now done in sim.configure_logging)
  • after all modules have been registered with the simulation
    • create a custom levels dictionary, only allowing WARNING messages for all disease modules, and then setting to get INFO messages for specific modules
    • call sim.configure_logging to set the logfile name and use the custom levels, assigning the output of this to keep the full logfile path.
  • remove your fh.flush() and fh.close(), this is already done in the simulation now
import datetime
import os
from pathlib import Path

from tlo import Date, Simulation
from tlo.analysis.utils import parse_log_file
from tlo.methods import (
    demography,
    enhanced_lifestyle,
    healthburden,
    healthsystem,
    hiv,
    logging,  # <--
    male_circumcision,
    tb,
)

# Where will outputs go
outputpath = Path("./outputs")  # folder for convenience of storing outputs

# date-stamp to label log files and any other outputs
datestamp = datetime.date.today().strftime("__%Y_%m_%d")

# The resource files
resourcefilepath = Path("./resources")

start_date = Date(2010, 1, 1)
end_date = Date(2015, 1, 1)
popsize = 5000

# Establish the simulation object
sim = Simulation(start_date=start_date)

# ----- Control over the types of intervention that can occur -----
# Make a list that contains the treatment_id that will be allowed. Empty list means nothing allowed.
# '*' means everything. It will allow any treatment_id that begins with a stub (e.g. Mockitis*)
service_availability = ["*"]

# Register the appropriate modules
sim.register(demography.Demography(resourcefilepath=resourcefilepath))
sim.register(healthsystem.HealthSystem(resourcefilepath=resourcefilepath))
sim.register(healthburden.HealthBurden(resourcefilepath=resourcefilepath))
sim.register(enhanced_lifestyle.Lifestyle(resourcefilepath=resourcefilepath))
sim.register(hiv.hiv(resourcefilepath=resourcefilepath))
sim.register(tb.tb(resourcefilepath=resourcefilepath))
sim.register(male_circumcision.male_circumcision(resourcefilepath=resourcefilepath))

# Sets all modules to WARNING threshold, then alters hiv, tb and male_circumcision to INFO
custom_levels = {"*": logging.WARNING,  # <--
                 "tlo.methods.hiv": logging.INFO,  # <--
                 "tlo.methods.tb": loggin.INFO,  # <--
                 "tlo.methods.male_circumcision": logging.INFO  # <--
}  # <--
# configure logging after registering modules with custom levels
logfile = sim.configure_logging(filename="LogFile", custom_levels=custom_levels)  # <--


# Run the simulation and flush the logger
sim.seed_rngs(0)
sim.make_initial_population(n=popsize)
sim.simulate(end_date=end_date)
# ...

TLO Model Wiki

Clone this wiki locally