docs/examples/notebooks/XML_ImportExport.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XML Import/Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NB: pip install pyhf[xmlio]\n",
    "import pyhf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 1752\r\n",
      "drwxr-xr-x   7 kratsg  staff   238B Oct 16 22:20 \u001b[34m.\u001b[m\u001b[m\r\n",
      "drwxr-xr-x  21 kratsg  staff   714B Apr  4 14:26 \u001b[34m..\u001b[m\u001b[m\r\n",
      "drwxr-xr-x   6 kratsg  staff   204B Feb 27 17:13 \u001b[34mconfig\u001b[m\u001b[m\r\n",
      "drwxr-xr-x   7 kratsg  staff   238B Feb 27 23:41 \u001b[34mdata\u001b[m\u001b[m\r\n",
      "-rw-r--r--   1 kratsg  staff   850K Oct 16 22:20 log\r\n",
      "drwxr-xr-x  17 kratsg  staff   578B Nov 15 12:24 \u001b[34mresults\u001b[m\u001b[m\r\n",
      "-rw-r--r--   1 kratsg  staff    21K Oct 16 22:20 scan.pdf\r\n"
     ]
    }
   ],
   "source": [
    "!ls -lavh ../../../validation/xmlimport_input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing\n",
    "\n",
    "In order to convert HistFactory XML+ROOT to the pyhf JSON spec for likelihoods, you need to point the command-line interface `pyhf xml2json` at the top-level XML file. Additionally, as the HistFactory XML specification often uses relative paths, you might need to specify the base directory `--basedir` from which all other files are located, as specified in the top-level XML. The command will be of the format\n",
    "\n",
    "```\n",
    "pyhf xml2json {top-level XML} --basedir {base directory}\n",
    "```\n",
    "\n",
    "This will print the JSON representation of the XML+ROOT specified. If you wish to store this as a JSON file, you simply need to redirect it\n",
    "\n",
    "```\n",
    "pyhf xml2json {top-level XML} --basedir {base directory} > spec.json\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\r\n",
      "    \"channels\": [\r\n",
      "        {\r\n",
      "            \"name\": \"channel1\",\r\n",
      "            \"samples\": [\r\n",
      "                {\r\n",
      "                    \"data\": [\r\n",
      "                        20.0,\r\n",
      "                        10.0\r\n",
      "                    ],\r\n",
      "                    \"modifiers\": [\r\n",
      "                        {\r\n",
      "                            \"data\": {\r\n",
      "                                \"hi\": 1.05,\r\n",
      "                                \"lo\": 0.95\r\n",
      "                            },\r\n",
      "                            \"name\": \"syst1\",\r\n",
      "                            \"type\": \"normsys\"\r\n",
      "                        },\r\n",
      "                        {\r\n",
      "                            \"data\": null,\r\n",
      "                            \"name\": \"SigXsecOverSM\",\r\n",
      "                            \"type\": \"normfactor\"\r\n",
      "                        }\r\n",
      "                    ],\r\n",
      "                    \"name\": \"signal\"\r\n",
      "                },\r\n",
      "                {\r\n",
      "                    \"data\": [\r\n",
      "                        100.0,\r\n",
      "                        0.0\r\n",
      "                    ],\r\n",
      "                    \"modifiers\": [\r",
      "\r\n",
      "                        {\r\n",
      "                            \"data\": null,\r\n",
      "                            \"name\": \"lumi\",\r\n",
      "                            \"type\": \"lumi\"\r\n",
      "                        },\r\n",
      "                        {\r\n",
      "                            \"data\": [\r\n",
      "                                5.000000074505806,\r\n",
      "                                0.0\r\n",
      "                            ],\r\n",
      "                            \"name\": \"staterror_channel1\",\r\n",
      "                            \"type\": \"staterror\"\r\n",
      "                        },\r\n",
      "                        {\r\n",
      "                            \"data\": {\r\n",
      "                                \"hi\": 1.05,\r\n",
      "                                \"lo\": 0.95\r\n",
      "                            },\r\n",
      "                            \"name\": \"syst2\",\r\n",
      "                            \"type\": \"normsys\"\r\n",
      "                        }\r\n",
      "                    ],\r\n",
      "                    \"name\": \"background1\"\r\n",
      "                },\r\n",
      "                {\r\n",
      "                    \"data\": [\r\n",
      "                        0.0,\r\n",
      "                        100.0\r\n",
      "                    ],\r\n",
      "                    \"modifiers\": [\r\n",
      "                        {\r\n",
      "                            \"data\": null,\r\n",
      "                            \"name\": \"lumi\",\r\n",
      "                            \"type\": \"lumi\"\r\n",
      "                        },\r\n",
      "                        {\r\n",
      "                            \"data\": [\r\n",
      "                                0.0,\r\n",
      "                                10.0\r\n",
      "                            ],\r\n",
      "                            \"name\": \"staterror_channel1\",\r\n",
      "                            \"type\": \"staterror\"\r\n",
      "                        },\r\n",
      "                        {\r\n",
      "                            \"data\": {\r\n",
      "                                \"hi\": 1.05,\r\n",
      "                                \"lo\": 0.95\r\n",
      "                            },\r\n",
      "                            \"name\": \"syst3\",\r\n",
      "                            \"type\": \"normsys\"\r\n",
      "                        }\r\n",
      "                    ],\r\n",
      "                    \"name\": \"background2\"\r\n",
      "                }\r\n",
      "            ]\r\n",
      "        }\r\n",
      "    ],\r\n",
      "    \"data\": {\r\n",
      "        \"channel1\": [\r\n",
      "            122.0,\r\n",
      "            112.0\r\n",
      "        ]\r\n",
      "    },\r\n",
      "    \"toplvl\": {\r\n",
      "        \"measurements\": [\r\n",
      "            {\r\n",
      "                \"config\": {\r\n",
      "                    \"parameters\": [\r\n",
      "                        {\r\n",
      "                            \"auxdata\": [\r\n",
      "                                1.0\r\n",
      "                            ],\r\n",
      "                            \"bounds\": [\r\n",
      "                                [\r\n",
      "                                    0.5,\r\n",
      "                                    1.5\r\n",
      "                                ]\r\n",
      "                            ],\r\n",
      "                            \"fixed\": true,\r\n",
      "                            \"inits\": [\r\n",
      "                                1.0\r\n",
      "                            ],\r\n",
      "                            \"name\": \"lumi\",\r\n",
      "                            \"sigmas\": [\r\n",
      "                                0.1\r\n",
      "                            ]\r\n",
      "                        },\r\n",
      "                        {\r\n",
      "                            \"fixed\": true,\r\n",
      "                            \"name\": \"alpha_syst1\"\r\n",
      "                        }\r\n",
      "                    ],\r\n",
      "                    \"poi\": \"SigXsecOverSM\"\r\n",
      "                },\r\n",
      "                \"name\": \"GaussExample\"\r\n",
      "            },\r\n",
      "            {\r\n",
      "                \"config\": {\r\n",
      "                    \"parameters\": [\r\n",
      "                        {\r\n",
      "                            \"auxdata\": [\r\n",
      "                                1.0\r\n",
      "                            ],\r\n",
      "                            \"bounds\": [\r\n",
      "                                [\r\n",
      "                                    0.5,\r\n",
      "                                    1.5\r\n",
      "                                ]\r\n",
      "                            ],\r\n",
      "                            \"fixed\": true,\r\n",
      "                            \"inits\": [\r\n",
      "                                1.0\r\n",
      "                            ],\r\n",
      "                            \"name\": \"lumi\",\r\n",
      "                            \"sigmas\": [\r\n",
      "                                0.1\r\n",
      "                            ]\r\n",
      "                        },\r\n",
      "                        {\r\n",
      "                            \"fixed\": true,\r\n",
      "                            \"name\": \"alpha_syst1\"\r\n",
      "                        }\r\n",
      "                    ],\r\n",
      "                    \"poi\": \"SigXsecOverSM\"\r\n",
      "                },\r\n",
      "                \"name\": \"GammaExample\"\r\n",
      "            },\r\n",
      "            {\r\n",
      "                \"config\": {\r\n",
      "                    \"parameters\": [\r\n",
      "                        {\r\n",
      "                            \"auxdata\": [\r\n",
      "                                1.0\r\n",
      "                            ],\r\n",
      "                            \"bounds\": [\r\n",
      "                                [\r\n",
      "                                    0.5,\r\n",
      "                                    1.5\r\n",
      "                                ]\r\n",
      "                            ],\r\n",
      "                            \"fixed\": true,\r\n",
      "                            \"inits\": [\r\n",
      "                                1.0\r\n",
      "                            ],\r\n",
      "                            \"name\": \"lumi\",\r\n",
      "                            \"sigmas\": [\r\n",
      "                                0.1\r\n",
      "                            ]\r\n",
      "                        },\r\n",
      "                        {\r\n",
      "                            \"fixed\": true,\r\n",
      "                            \"name\": \"alpha_syst1\"\r\n",
      "                        }\r\n",
      "                    ],\r\n",
      "                    \"poi\": \"SigXsecOverSM\"\r\n",
      "                },\r\n",
      "                \"name\": \"LogNormExample\"\r\n",
      "            },\r\n",
      "            {\r\n",
      "                \"config\": {\r\n",
      "                    \"parameters\": [\r\n",
      "                        {\r\n",
      "                            \"auxdata\": [\r\n",
      "                                1.0\r\n",
      "                            ],\r\n",
      "                            \"bounds\": [\r\n",
      "                                [\r\n",
      "                                    0.5,\r\n",
      "                                    1.5\r\n",
      "                                ]\r\n",
      "                            ],\r\n",
      "                            \"fixed\": true,\r\n",
      "                            \"inits\": [\r\n",
      "                                1.0\r\n",
      "                            ],\r\n",
      "                            \"name\": \"lumi\",\r\n",
      "                            \"sigmas\": [\r\n",
      "                                0.1\r\n",
      "                            ]\r\n",
      "                        },\r\n",
      "                        {\r\n",
      "                            \"fixed\": true,\r\n",
      "                            \"name\": \"alpha_syst1\"\r\n",
      "                        }\r\n",
      "                    ],\r\n",
      "                    \"poi\": \"SigXsecOverSM\"\r\n",
      "                },\r\n",
      "                \"name\": \"ConstExample\"\r\n",
      "            }\r\n",
      "        ],\r\n",
      "        \"resultprefix\": \"./results/example\"\r\n",
      "    }\r\n",
      "}\r\n"
     ]
    }
   ],
   "source": [
    "!pyhf xml2json --hide-progress ../../../validation/xmlimport_input/config/example.xml --basedir ../../../validation/xmlimport_input | tee xml_importexport.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exporting\n",
    "\n",
    "In order to convert the pyhf JSON to the HistFactory XML+ROOT spec for likelihoods, you need to point the command-line interface `pyhf json2xml` at the JSON file you want to convert. As everything is specified in a single file, there is no need to deal with base directories or looking up additional files. This will produce output XML+ROOT in the `--output-dir=./` directory (your current working directory), storing XML configs under `--specroot=config` and the data file under `--dataroot=data`. The XML configs are prefixed with `--resultprefix=FitConfig` by default, so that the top-level XML file will be located at `{output dir}/{prefix}.xml`. The command will be of the format\n",
    "\n",
    "```\n",
    "pyhf json2xml {JSON spec}\n",
    "```\n",
    "\n",
    "Note that the output directory must already exist."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/jovyan/pyhf/src/pyhf/writexml.py:120: RuntimeWarning: invalid value encountered in true_divide\n",
      "  attrs['HistoName'], np.divide(modifierspec['data'], sampledata).tolist()\n",
      "-rw-r--r--  1 kratsg  staff   822B Apr  9 09:36 output/FitConfig.xml\n",
      "\n",
      "output/config:\n",
      "total 8\n",
      "drwxr-xr-x  3 kratsg  staff   102B Apr  9 09:36 \u001b[34m.\u001b[m\u001b[m\n",
      "drwxr-xr-x  5 kratsg  staff   170B Apr  9 09:36 \u001b[34m..\u001b[m\u001b[m\n",
      "-rw-r--r--  1 kratsg  staff   1.0K Apr  9 09:36 FitConfig_channel1.xml\n",
      "\n",
      "output/data:\n",
      "total 96\n",
      "drwxr-xr-x  3 kratsg  staff   102B Apr  9 09:36 \u001b[34m.\u001b[m\u001b[m\n",
      "drwxr-xr-x  5 kratsg  staff   170B Apr  9 09:36 \u001b[34m..\u001b[m\u001b[m\n",
      "-rw-r--r--  1 kratsg  staff    46K Apr  9 09:36 data.root\n"
     ]
    }
   ],
   "source": [
    "!mkdir -p output\n",
    "!pyhf json2xml xml_importexport.json --output-dir output\n",
    "!ls -lavh output/*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm xml_importexport.json\n",
    "!rm -rf output/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}