01_simulate_scenarios_1_2_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "694dfd52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thu Nov 30 15:48:44 PST 2023\r\n"
     ]
    }
   ],
   "source": [
    "import numpy as np, matplotlib.pyplot as plt, pandas as pd\n",
    "pd.set_option('display.max_rows', 8)\n",
    "!date\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df74a1e9",
   "metadata": {},
   "source": [
    "# Scenario 1, 2, 3: non-TDA approaches to DAS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8432d3b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(12345)  # set random seed for reproducibility"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc70badb",
   "metadata": {},
   "source": [
    "# Load synthetic data for TX and use it to simulate 2010 and 2020 populations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "feaa2385",
   "metadata": {},
   "outputs": [],
   "source": [
    "import linked_census_disclosure.data as lcd_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a3e291fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 45s, sys: 28.2 s, total: 2min 13s\n",
      "Wall time: 2min 13s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df = lcd_data.read_synth_data('tx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6265d968",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'25,145,561'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# does this have the expected number of rows?\n",
    "f'{len(df):,.0f}' # expect population of texas in 2010 to be 25,145,561"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad4e2b13",
   "metadata": {},
   "source": [
    "# Focus in on the 10-17 year olds\n",
    "\n",
    "We will use this group as our 2020 population and then simulate what their attributes were in 2010 for our 2010 population."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bd2eb411",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3,009,117'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all = df\n",
    "df = df[(df.age >= 10) & (df.age < 18)].copy()\n",
    "\n",
    "n_kids = len(df)  # number of children\n",
    "f'{n_kids:,.0f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "913f6057",
   "metadata": {},
   "outputs": [],
   "source": [
    "#### simulate 10 years of demographic change\n",
    "\n",
    "df['age_2020'] = df.age\n",
    "df['age_2010'] = df.age - 10\n",
    "del df['age']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "c6ee0d67",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert np.all(df.age_2010 >= 0), 'ensure that all ages are still non-negative'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "275dab3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ignore births, because we are focused\n",
    "# only on kids who can be linked between 2010 and 2020 census"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31fa3364",
   "metadata": {},
   "source": [
    "# Add in migration, make the change to the 2010 geography\n",
    "\n",
    "p_stay from ACS, see [2022_04_19a_das_dhc_attack_mig_data.ipynb](2022_04_19a_das_dhc_attack_mig_data.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cb7e14cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# simple model migration, based on probability\n",
    "# of being in same house for 10+ years among household with 8-17 year olds\n",
    "# in ACS\n",
    "\n",
    "p_stay = 0.23\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bc461fd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_locations = df_all.hh_id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3dda86af",
   "metadata": {},
   "outputs": [],
   "source": [
    "locations_2020 = df.hh_id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "03f7e783",
   "metadata": {},
   "outputs": [],
   "source": [
    "random_location = np.random.choice(all_locations, size=len(locations_2020),\n",
    "                     replace=True)\n",
    "\n",
    "locations_2010 = np.where(np.random.uniform(size=len(locations_2020)) < p_stay,\n",
    "                         locations_2020,\n",
    "                         random_location)\n",
    "\n",
    "s_location_2010 = pd.Series(locations_2010,\n",
    "                            index=locations_2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7b83fc07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22957601574075331"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(locations_2010 == locations_2020)  # should be around 0.23"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ab0abe43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>state</th>\n",
       "      <th>county</th>\n",
       "      <th>tract</th>\n",
       "      <th>block</th>\n",
       "      <th>geoid</th>\n",
       "      <th>sex_id</th>\n",
       "      <th>relationship</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>racaian</th>\n",
       "      <th>racasn</th>\n",
       "      <th>racblk</th>\n",
       "      <th>racnhpi</th>\n",
       "      <th>racsor</th>\n",
       "      <th>racwht</th>\n",
       "      <th>pweight</th>\n",
       "      <th>age_2020</th>\n",
       "      <th>age_2010</th>\n",
       "      <th>hh_id_2020</th>\n",
       "      <th>hh_id_2010</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>48</td>\n",
       "      <td>61</td>\n",
       "      <td>980100</td>\n",
       "      <td>1000</td>\n",
       "      <td>480619801001000</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>480619801001000-1</td>\n",
       "      <td>483090037102003-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>48</td>\n",
       "      <td>61</td>\n",
       "      <td>980100</td>\n",
       "      <td>1000</td>\n",
       "      <td>480619801001000</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>480619801001000-2</td>\n",
       "      <td>484790012011000-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>48</td>\n",
       "      <td>61</td>\n",
       "      <td>980100</td>\n",
       "      <td>1000</td>\n",
       "      <td>480619801001000</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>480619801001000-2</td>\n",
       "      <td>484790012011000-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>48</td>\n",
       "      <td>441</td>\n",
       "      <td>11600</td>\n",
       "      <td>1000</td>\n",
       "      <td>484410116001000</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>3</td>\n",
       "      <td>484410116001000-52</td>\n",
       "      <td>484410116001000-52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25145326</th>\n",
       "      <td>48</td>\n",
       "      <td>113</td>\n",
       "      <td>7916</td>\n",
       "      <td>2001</td>\n",
       "      <td>481130079162001</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>481130079162001-743</td>\n",
       "      <td>483396940011002-29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25145453</th>\n",
       "      <td>48</td>\n",
       "      <td>113</td>\n",
       "      <td>7916</td>\n",
       "      <td>1002</td>\n",
       "      <td>481130079161002</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>481130079161002-738</td>\n",
       "      <td>484910215152000-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25145494</th>\n",
       "      <td>48</td>\n",
       "      <td>113</td>\n",
       "      <td>7916</td>\n",
       "      <td>2001</td>\n",
       "      <td>481130079162001</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>481130079162001-788</td>\n",
       "      <td>484910201171011-2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25145495</th>\n",
       "      <td>48</td>\n",
       "      <td>113</td>\n",
       "      <td>7916</td>\n",
       "      <td>2001</td>\n",
       "      <td>481130079162001</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>481130079162001-791</td>\n",
       "      <td>481410041031009-356</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3009117 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          state  county   tract  block            geoid  sex_id  relationship  \\\n",
       "2            48      61  980100   1000  480619801001000       1            25   \n",
       "6            48      61  980100   1000  480619801001000       2            25   \n",
       "7            48      61  980100   1000  480619801001000       2            25   \n",
       "16           48     441   11600   1000  484410116001000       1            20   \n",
       "...         ...     ...     ...    ...              ...     ...           ...   \n",
       "25145326     48     113    7916   2001  481130079162001       2            22   \n",
       "25145453     48     113    7916   1002  481130079161002       2            28   \n",
       "25145494     48     113    7916   2001  481130079162001       1            25   \n",
       "25145495     48     113    7916   2001  481130079162001       1            36   \n",
       "\n",
       "          hispanic  racaian  racasn  racblk  racnhpi  racsor  racwht  pweight  \\\n",
       "2                1        0       0       0        0       0       1        1   \n",
       "6                1        0       0       0        0       0       1        1   \n",
       "7                1        0       0       0        0       0       1        1   \n",
       "16               0        0       0       0        0       0       1        1   \n",
       "...            ...      ...     ...     ...      ...     ...     ...      ...   \n",
       "25145326         0        0       1       0        0       0       0        1   \n",
       "25145453         1        0       0       0        0       0       1        1   \n",
       "25145494         1        0       0       0        0       1       0        1   \n",
       "25145495         1        0       0       0        0       1       0        1   \n",
       "\n",
       "          age_2020  age_2010           hh_id_2020           hh_id_2010  \n",
       "2               15         5    480619801001000-1    483090037102003-1  \n",
       "6               10         0    480619801001000-2   484790012011000-16  \n",
       "7               15         5    480619801001000-2   484790012011000-16  \n",
       "16              13         3   484410116001000-52   484410116001000-52  \n",
       "...            ...       ...                  ...                  ...  \n",
       "25145326        17         7  481130079162001-743   483396940011002-29  \n",
       "25145453        14         4  481130079161002-738   484910215152000-16  \n",
       "25145494        10         0  481130079162001-788    484910201171011-2  \n",
       "25145495        10         0  481130079162001-791  481410041031009-356  \n",
       "\n",
       "[3009117 rows x 19 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['hh_id_2020'] = df.hh_id\n",
    "df['hh_id_2010'] = df.hh_id.map(s_location_2010)\n",
    "del df['hh_id']\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "23ca3a45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22982755406320193"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(df.hh_id_2010 == df.hh_id_2020)  # should be around 23%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bceae15f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['geoid_2020'] = df.geoid\n",
    "df['geoid_2010'] = df.hh_id_2010.map(lambda x: int(x.split('-')[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f068bc67",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.22983353588444716"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(df.geoid_2010 == df.geoid_2020)  # should be around 23%"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c5eb6f4",
   "metadata": {},
   "source": [
    "# Make gender column, based on BRFSS 2019 SOGI results\n",
    "\n",
    "In future work, could try to incorporate observation that there is substantial age dependence in these values.  But for now, keep it simple, and work with the crude prevalence rates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e6db3c8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['reported_sex_2020'] = df.sex_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "612b2d85",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.00182556, 0.00223124, 0.00121704, 0.99472617])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# then initalize a gender for each simulant, calibrated to have unconditional probability from BRFSS\n",
    "# and also to match the data generation procedure for the reported_sex_2020 column\n",
    "\n",
    "p_trans_boy = 0.18 / 100\n",
    "p_trans_girl = 0.22 / 100\n",
    "p_trans_other = 0.12 / 100\n",
    "p_cis = 98.08 / 100\n",
    "\n",
    "# rescale to sum to 100%\n",
    "p_gender = np.array([p_trans_boy, p_trans_girl, p_trans_other, p_cis])\n",
    "p_gender /= p_gender.sum()\n",
    "p_gender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fe3d9511",
   "metadata": {},
   "outputs": [],
   "source": [
    "# first initialize gender without distinguishing cis boy and cis girl\n",
    "# since that matches BRFSS SOGI question\n",
    "df['gender'] = np.random.choice(['trans_boy', 'trans_girl', 'trans_other', 'cis'], p=p_gender, size=len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "18f988d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# now distinguish cis based on reconstructed sex_id\n",
    "df.gender = np.where(df.gender == 'cis',\n",
    "                     df.sex_id.map({1:'cis_boy', 2:'cis_girl'}),\n",
    "                     df.gender  # ~50% of trans_boys have reported_sex_2020 female, etc\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "682973de",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['trans'] = df.gender.isin(['trans_boy', 'trans_girl', 'trans_other'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "44c3bdc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cis_boy        50.98\n",
       "cis_girl       48.49\n",
       "trans_girl      0.23\n",
       "trans_boy       0.18\n",
       "trans_other     0.12\n",
       "Name: gender, dtype: float64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(100 * df.gender.value_counts(normalize=True), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d24aba4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.53"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(100 * df.gender.value_counts(normalize=True).filter(like='trans').sum(), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "bc986237",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.53"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(100 * df.trans.mean(), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7aae5ed4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>reported_sex_2020</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>trans_boy</th>\n",
       "      <td>0.509158</td>\n",
       "      <td>0.490842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trans_girl</th>\n",
       "      <td>0.512911</td>\n",
       "      <td>0.487089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trans_other</th>\n",
       "      <td>0.525170</td>\n",
       "      <td>0.474830</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "reported_sex_2020         1         2\n",
       "gender                               \n",
       "trans_boy          0.509158  0.490842\n",
       "trans_girl         0.512911  0.487089\n",
       "trans_other        0.525170  0.474830"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.trans].groupby('gender').reported_sex_2020.value_counts(normalize=True).unstack()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c8329428",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gender_to_sex_2010(gender, reported_sex_2020):\n",
    "    # start with values reported in 2020\n",
    "    sex = reported_sex_2020.copy()\n",
    "    \n",
    "    # update the trans_boy and trans_girl entries to be sex assigned at birth\n",
    "    sex = np.where((gender == 'trans_boy'),\n",
    "                   2,\n",
    "                   sex\n",
    "                  )\n",
    "    sex = np.where((gender == 'trans_girl'),\n",
    "                   1,\n",
    "                   sex\n",
    "                  )\n",
    "    return sex\n",
    "\n",
    "\n",
    "df['reported_sex_2010'] = gender_to_sex_2010(df.gender, df.reported_sex_2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "cf41dbc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0020271727553298857"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(df.reported_sex_2010 != df.reported_sex_2020)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8ff9b64",
   "metadata": {},
   "source": [
    "# Values for results section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e7143663",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3,009,117'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Our synthetic population matched the age, sex, race/ethnicity, and geography of Texas\n",
    "# on census day April 1, 2010, with\n",
    "# X children ages 0-7 in Y household on census day 2010\n",
    "\n",
    "n_kids = len(df)  # number of children\n",
    "f'{n_kids:,.0f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "03dce5de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6168.689850000001"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# back of envelope scenario 1\n",
    "3_009_117 * (0.18 + 0.23)/100 * 0.5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3d44190c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1384.1938200000002"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# back of envelope scenario 2\n",
    "3_009_117 * 0.002 * 0.23 # * fraction unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ed9e5907",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2,354,144'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_households = df.hh_id_2020.nunique()  # number of households\n",
    "f'{n_households:,.0f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "fa0bd1ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'691,578'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of household that were in same place in 2010 and 2020 census\n",
    "n_stayed = (df.hh_id_2010 == df.hh_id_2020).sum()\n",
    "f'{n_stayed:,.0f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "8e65af6b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3,625'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of trans kids that were in same place in 2010 and 2020 census\n",
    "\n",
    "n_trans_stayed = (df.trans & (df.hh_id_2010 == df.hh_id_2020)).sum()\n",
    "f'{n_trans_stayed:,.0f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9710d59b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'6,090'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of trans families identified if full census data with names and dob was released\n",
    "\n",
    "n_hh_w_sex_different = df[df.reported_sex_2010 != df.reported_sex_2020].hh_id_2010.nunique()\n",
    "f'{n_hh_w_sex_different:,.0f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "54592c52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'6,100'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of trans kids identified if full census data with names and dob was released\n",
    "\n",
    "n_kids_w_sex_different = len(df[df.reported_sex_2010 != df.reported_sex_2020])\n",
    "f'{n_kids_w_sex_different:,.0f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9933111f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'15,951'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of trans kids total\n",
    "\n",
    "n_trans_kids = sum(df.trans)\n",
    "f'{n_trans_kids:,.0f}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "21e8de9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38.0"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(100 * n_kids_w_sex_different/n_trans_kids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "594fdcaf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>state</th>\n",
       "      <th>county</th>\n",
       "      <th>tract</th>\n",
       "      <th>block</th>\n",
       "      <th>sex_id</th>\n",
       "      <th>relationship</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>racaian</th>\n",
       "      <th>racasn</th>\n",
       "      <th>racblk</th>\n",
       "      <th>...</th>\n",
       "      <th>age_2020</th>\n",
       "      <th>age_2010</th>\n",
       "      <th>hh_id_2020</th>\n",
       "      <th>hh_id_2010</th>\n",
       "      <th>geoid_2020</th>\n",
       "      <th>geoid_2010</th>\n",
       "      <th>reported_sex_2020</th>\n",
       "      <th>gender</th>\n",
       "      <th>trans</th>\n",
       "      <th>reported_sex_2010</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>48</td>\n",
       "      <td>61</td>\n",
       "      <td>980100</td>\n",
       "      <td>1000</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>480619801001000-1</td>\n",
       "      <td>483090037102003-1</td>\n",
       "      <td>480619801001000</td>\n",
       "      <td>483090037102003</td>\n",
       "      <td>1</td>\n",
       "      <td>cis_boy</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>48</td>\n",
       "      <td>61</td>\n",
       "      <td>980100</td>\n",
       "      <td>1000</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>480619801001000-2</td>\n",
       "      <td>484790012011000-16</td>\n",
       "      <td>480619801001000</td>\n",
       "      <td>484790012011000</td>\n",
       "      <td>2</td>\n",
       "      <td>cis_girl</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>48</td>\n",
       "      <td>61</td>\n",
       "      <td>980100</td>\n",
       "      <td>1000</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>480619801001000-2</td>\n",
       "      <td>484790012011000-16</td>\n",
       "      <td>480619801001000</td>\n",
       "      <td>484790012011000</td>\n",
       "      <td>2</td>\n",
       "      <td>cis_girl</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>48</td>\n",
       "      <td>441</td>\n",
       "      <td>11600</td>\n",
       "      <td>1000</td>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>13</td>\n",
       "      <td>3</td>\n",
       "      <td>484410116001000-52</td>\n",
       "      <td>484410116001000-52</td>\n",
       "      <td>484410116001000</td>\n",
       "      <td>484410116001000</td>\n",
       "      <td>1</td>\n",
       "      <td>cis_boy</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25145326</th>\n",
       "      <td>48</td>\n",
       "      <td>113</td>\n",
       "      <td>7916</td>\n",
       "      <td>2001</td>\n",
       "      <td>2</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>17</td>\n",
       "      <td>7</td>\n",
       "      <td>481130079162001-743</td>\n",
       "      <td>483396940011002-29</td>\n",
       "      <td>481130079162001</td>\n",
       "      <td>483396940011002</td>\n",
       "      <td>2</td>\n",
       "      <td>cis_girl</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25145453</th>\n",
       "      <td>48</td>\n",
       "      <td>113</td>\n",
       "      <td>7916</td>\n",
       "      <td>1002</td>\n",
       "      <td>2</td>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>481130079161002-738</td>\n",
       "      <td>484910215152000-16</td>\n",
       "      <td>481130079161002</td>\n",
       "      <td>484910215152000</td>\n",
       "      <td>2</td>\n",
       "      <td>cis_girl</td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25145494</th>\n",
       "      <td>48</td>\n",
       "      <td>113</td>\n",
       "      <td>7916</td>\n",
       "      <td>2001</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>481130079162001-788</td>\n",
       "      <td>484910201171011-2</td>\n",
       "      <td>481130079162001</td>\n",
       "      <td>484910201171011</td>\n",
       "      <td>1</td>\n",
       "      <td>cis_boy</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25145495</th>\n",
       "      <td>48</td>\n",
       "      <td>113</td>\n",
       "      <td>7916</td>\n",
       "      <td>2001</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>481130079162001-791</td>\n",
       "      <td>481410041031009-356</td>\n",
       "      <td>481130079162001</td>\n",
       "      <td>481410041031009</td>\n",
       "      <td>1</td>\n",
       "      <td>cis_boy</td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3009117 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          state  county   tract  block  sex_id  relationship  hispanic  \\\n",
       "2            48      61  980100   1000       1            25         1   \n",
       "6            48      61  980100   1000       2            25         1   \n",
       "7            48      61  980100   1000       2            25         1   \n",
       "16           48     441   11600   1000       1            20         0   \n",
       "...         ...     ...     ...    ...     ...           ...       ...   \n",
       "25145326     48     113    7916   2001       2            22         0   \n",
       "25145453     48     113    7916   1002       2            28         1   \n",
       "25145494     48     113    7916   2001       1            25         1   \n",
       "25145495     48     113    7916   2001       1            36         1   \n",
       "\n",
       "          racaian  racasn  racblk  ...  age_2020  age_2010  \\\n",
       "2               0       0       0  ...        15         5   \n",
       "6               0       0       0  ...        10         0   \n",
       "7               0       0       0  ...        15         5   \n",
       "16              0       0       0  ...        13         3   \n",
       "...           ...     ...     ...  ...       ...       ...   \n",
       "25145326        0       1       0  ...        17         7   \n",
       "25145453        0       0       0  ...        14         4   \n",
       "25145494        0       0       0  ...        10         0   \n",
       "25145495        0       0       0  ...        10         0   \n",
       "\n",
       "                   hh_id_2020           hh_id_2010       geoid_2020  \\\n",
       "2           480619801001000-1    483090037102003-1  480619801001000   \n",
       "6           480619801001000-2   484790012011000-16  480619801001000   \n",
       "7           480619801001000-2   484790012011000-16  480619801001000   \n",
       "16         484410116001000-52   484410116001000-52  484410116001000   \n",
       "...                       ...                  ...              ...   \n",
       "25145326  481130079162001-743   483396940011002-29  481130079162001   \n",
       "25145453  481130079161002-738   484910215152000-16  481130079161002   \n",
       "25145494  481130079162001-788    484910201171011-2  481130079162001   \n",
       "25145495  481130079162001-791  481410041031009-356  481130079162001   \n",
       "\n",
       "               geoid_2010 reported_sex_2020    gender  trans  \\\n",
       "2         483090037102003                 1   cis_boy  False   \n",
       "6         484790012011000                 2  cis_girl  False   \n",
       "7         484790012011000                 2  cis_girl  False   \n",
       "16        484410116001000                 1   cis_boy  False   \n",
       "...                   ...               ...       ...    ...   \n",
       "25145326  483396940011002                 2  cis_girl  False   \n",
       "25145453  484910215152000                 2  cis_girl  False   \n",
       "25145494  484910201171011                 1   cis_boy  False   \n",
       "25145495  481410041031009                 1   cis_boy  False   \n",
       "\n",
       "          reported_sex_2010  \n",
       "2                         1  \n",
       "6                         2  \n",
       "7                         2  \n",
       "16                        1  \n",
       "...                     ...  \n",
       "25145326                  2  \n",
       "25145453                  2  \n",
       "25145494                  1  \n",
       "25145495                  1  \n",
       "\n",
       "[3009117 rows x 24 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df['geoid']\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "b59e94fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>n_simulants</th>\n",
       "      <th>pct_male</th>\n",
       "      <th>pct_trans</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>geoid</th>\n",
       "      <th>age</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>racwht</th>\n",
       "      <th>racblk</th>\n",
       "      <th>racasn</th>\n",
       "      <th>racaian</th>\n",
       "      <th>racnhpi</th>\n",
       "      <th>racsor</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">480019501001000</th>\n",
       "      <th>16</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">480019501001001</th>\n",
       "      <th>11</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">485079503025024</th>\n",
       "      <th>14</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2211007 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                          n_simulants  \\\n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor                \n",
       "480019501001000 16  0        0      1      0      0       0       0                 1   \n",
       "                17  0        1      0      0      0       0       0                 1   \n",
       "480019501001001 11  0        0      1      0      0       0       0                 1   \n",
       "                13  0        1      0      0      0       0       0                 1   \n",
       "...                                                                               ...   \n",
       "485079503025024 14  1        1      0      0      0       0       0                 1   \n",
       "                15  1        1      0      0      0       0       1                 1   \n",
       "                16  1        1      0      0      0       0       0                 1   \n",
       "                17  1        1      0      0      0       0       0                 1   \n",
       "\n",
       "                                                                          pct_male  \\\n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor             \n",
       "480019501001000 16  0        0      1      0      0       0       0            0.0   \n",
       "                17  0        1      0      0      0       0       0          100.0   \n",
       "480019501001001 11  0        0      1      0      0       0       0          100.0   \n",
       "                13  0        1      0      0      0       0       0          100.0   \n",
       "...                                                                            ...   \n",
       "485079503025024 14  1        1      0      0      0       0       0          100.0   \n",
       "                15  1        1      0      0      0       0       1            0.0   \n",
       "                16  1        1      0      0      0       0       0            0.0   \n",
       "                17  1        1      0      0      0       0       0          100.0   \n",
       "\n",
       "                                                                          pct_trans  \n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor             \n",
       "480019501001000 16  0        0      1      0      0       0       0             0.0  \n",
       "                17  0        1      0      0      0       0       0             0.0  \n",
       "480019501001001 11  0        0      1      0      0       0       0             0.0  \n",
       "                13  0        1      0      0      0       0       0             0.0  \n",
       "...                                                                             ...  \n",
       "485079503025024 14  1        1      0      0      0       0       0             0.0  \n",
       "                15  1        1      0      0      0       0       1             0.0  \n",
       "                16  1        1      0      0      0       0       0             0.0  \n",
       "                17  1        1      0      0      0       0       0             0.0  \n",
       "\n",
       "[2211007 rows x 3 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# without reidentification to link on, there is still a risk of identifying a block with a trans kid\n",
    "# by finding blocks where there was a single kids of a given age in 2010 and that age+10 in 2020 and\n",
    "# different reported sex\n",
    "df['reported_male_2010'] = (df.reported_sex_2010 == 1)  # column for easily calculating percet male in each strata\n",
    "df['geoid'] = df.geoid_2010\n",
    "df['age'] = df.age_2010 + 10 # add ten to make merge easier\n",
    "\n",
    "g = df.groupby(['geoid', 'age',\n",
    "                'hispanic', 'racwht', 'racblk',\n",
    "                'racasn', 'racaian', 'racnhpi', 'racsor'])\n",
    "\n",
    "df_a = pd.DataFrame({'n_simulants': g.pweight.sum()})\n",
    "df_a['pct_male'] = 100*g.reported_male_2010.mean()\n",
    "df_a['pct_trans'] = 100*g.trans.mean()\n",
    "df_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "90a71ea8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1722991"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the rows with n_simulants == 1 are all strata with a single simulant in 2010\n",
    "# (I used age_2020 in the index to make it easier to merge them with the df_b I will construct next)\n",
    "n_unique_2010 = sum(df_a.n_simulants == 1)\n",
    "n_unique_2010"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d0b08c7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1413801"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['reported_male_2020'] = (df.reported_sex_2020 == 1)\n",
    "df['geoid'] = df.geoid_2020\n",
    "df['age'] = df.age_2020\n",
    "df['discordant_sex'] = (df.reported_sex_2010 != df.reported_sex_2020)\n",
    "g = df.groupby(['geoid', 'age',\n",
    "                'hispanic', 'racwht', 'racblk',\n",
    "                'racasn', 'racaian', 'racnhpi', 'racsor'])\n",
    "\n",
    "df_b = pd.DataFrame({'n_simulants': g.pweight.sum()})\n",
    "df_b['pct_male'] = 100*g.reported_male_2020.mean()\n",
    "df_b['pct_trans'] = 100*g.trans.mean()\n",
    "df_b['pct_discordant_sex'] = 100*g.discordant_sex.mean()\n",
    "n_unique_2020 = sum(df_b.n_simulants == 1)\n",
    "n_unique_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "2170d0b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ab = df_a[df_a.n_simulants==1].copy()\n",
    "df_ab.columns = ['n_simulants_2010', 'pct_male_2010', 'pct_trans_2010']\n",
    "\n",
    "df_ab['n_simulants_2020'] = df_b[df_b.n_simulants==1].n_simulants\n",
    "df_ab['pct_male_2020'] = df_b[df_b.n_simulants==1].pct_male\n",
    "df_ab['pct_trans_2020'] = df_b[df_b.n_simulants==1].pct_trans\n",
    "df_ab['pct_discordant_sex'] = df_b[df_b.n_simulants==1].pct_discordant_sex\n",
    "df_ab = df_ab.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "d6137fc8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>n_simulants_2010</th>\n",
       "      <th>pct_male_2010</th>\n",
       "      <th>pct_trans_2010</th>\n",
       "      <th>n_simulants_2020</th>\n",
       "      <th>pct_male_2020</th>\n",
       "      <th>pct_trans_2020</th>\n",
       "      <th>pct_discordant_sex</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>geoid</th>\n",
       "      <th>age</th>\n",
       "      <th>hispanic</th>\n",
       "      <th>racwht</th>\n",
       "      <th>racblk</th>\n",
       "      <th>racasn</th>\n",
       "      <th>racaian</th>\n",
       "      <th>racnhpi</th>\n",
       "      <th>racsor</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>480019501001022</th>\n",
       "      <th>15</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480019501002073</th>\n",
       "      <th>16</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480019501002086</th>\n",
       "      <th>11</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480019501002112</th>\n",
       "      <th>11</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485079503025006</th>\n",
       "      <th>12</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485079503025008</th>\n",
       "      <th>13</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485079503025017</th>\n",
       "      <th>17</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485079503025023</th>\n",
       "      <th>16</th>\n",
       "      <th>1</th>\n",
       "      <th>1</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>69959 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                          n_simulants_2010  \\\n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor                     \n",
       "480019501001022 15  0        0      1      0      0       0       0                      1   \n",
       "480019501002073 16  0        1      0      0      0       0       0                      1   \n",
       "480019501002086 11  0        1      0      0      0       0       0                      1   \n",
       "480019501002112 11  0        1      0      0      0       0       0                      1   \n",
       "...                                                                                    ...   \n",
       "485079503025006 12  1        1      0      0      0       0       0                      1   \n",
       "485079503025008 13  1        1      0      0      0       0       0                      1   \n",
       "485079503025017 17  1        1      0      0      0       0       0                      1   \n",
       "485079503025023 16  1        1      0      0      0       0       0                      1   \n",
       "\n",
       "                                                                          pct_male_2010  \\\n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor                  \n",
       "480019501001022 15  0        0      1      0      0       0       0                 0.0   \n",
       "480019501002073 16  0        1      0      0      0       0       0                 0.0   \n",
       "480019501002086 11  0        1      0      0      0       0       0               100.0   \n",
       "480019501002112 11  0        1      0      0      0       0       0                 0.0   \n",
       "...                                                                                 ...   \n",
       "485079503025006 12  1        1      0      0      0       0       0               100.0   \n",
       "485079503025008 13  1        1      0      0      0       0       0                 0.0   \n",
       "485079503025017 17  1        1      0      0      0       0       0               100.0   \n",
       "485079503025023 16  1        1      0      0      0       0       0               100.0   \n",
       "\n",
       "                                                                          pct_trans_2010  \\\n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor                   \n",
       "480019501001022 15  0        0      1      0      0       0       0                  0.0   \n",
       "480019501002073 16  0        1      0      0      0       0       0                  0.0   \n",
       "480019501002086 11  0        1      0      0      0       0       0                  0.0   \n",
       "480019501002112 11  0        1      0      0      0       0       0                  0.0   \n",
       "...                                                                                  ...   \n",
       "485079503025006 12  1        1      0      0      0       0       0                  0.0   \n",
       "485079503025008 13  1        1      0      0      0       0       0                  0.0   \n",
       "485079503025017 17  1        1      0      0      0       0       0                  0.0   \n",
       "485079503025023 16  1        1      0      0      0       0       0                  0.0   \n",
       "\n",
       "                                                                          n_simulants_2020  \\\n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor                     \n",
       "480019501001022 15  0        0      1      0      0       0       0                    1.0   \n",
       "480019501002073 16  0        1      0      0      0       0       0                    1.0   \n",
       "480019501002086 11  0        1      0      0      0       0       0                    1.0   \n",
       "480019501002112 11  0        1      0      0      0       0       0                    1.0   \n",
       "...                                                                                    ...   \n",
       "485079503025006 12  1        1      0      0      0       0       0                    1.0   \n",
       "485079503025008 13  1        1      0      0      0       0       0                    1.0   \n",
       "485079503025017 17  1        1      0      0      0       0       0                    1.0   \n",
       "485079503025023 16  1        1      0      0      0       0       0                    1.0   \n",
       "\n",
       "                                                                          pct_male_2020  \\\n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor                  \n",
       "480019501001022 15  0        0      1      0      0       0       0               100.0   \n",
       "480019501002073 16  0        1      0      0      0       0       0               100.0   \n",
       "480019501002086 11  0        1      0      0      0       0       0                 0.0   \n",
       "480019501002112 11  0        1      0      0      0       0       0               100.0   \n",
       "...                                                                                 ...   \n",
       "485079503025006 12  1        1      0      0      0       0       0                 0.0   \n",
       "485079503025008 13  1        1      0      0      0       0       0               100.0   \n",
       "485079503025017 17  1        1      0      0      0       0       0                 0.0   \n",
       "485079503025023 16  1        1      0      0      0       0       0                 0.0   \n",
       "\n",
       "                                                                          pct_trans_2020  \\\n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor                   \n",
       "480019501001022 15  0        0      1      0      0       0       0                  0.0   \n",
       "480019501002073 16  0        1      0      0      0       0       0                  0.0   \n",
       "480019501002086 11  0        1      0      0      0       0       0                  0.0   \n",
       "480019501002112 11  0        1      0      0      0       0       0                  0.0   \n",
       "...                                                                                  ...   \n",
       "485079503025006 12  1        1      0      0      0       0       0                  0.0   \n",
       "485079503025008 13  1        1      0      0      0       0       0                  0.0   \n",
       "485079503025017 17  1        1      0      0      0       0       0                  0.0   \n",
       "485079503025023 16  1        1      0      0      0       0       0                  0.0   \n",
       "\n",
       "                                                                          pct_discordant_sex  \n",
       "geoid           age hispanic racwht racblk racasn racaian racnhpi racsor                      \n",
       "480019501001022 15  0        0      1      0      0       0       0                      0.0  \n",
       "480019501002073 16  0        1      0      0      0       0       0                      0.0  \n",
       "480019501002086 11  0        1      0      0      0       0       0                      0.0  \n",
       "480019501002112 11  0        1      0      0      0       0       0                      0.0  \n",
       "...                                                                                      ...  \n",
       "485079503025006 12  1        1      0      0      0       0       0                      0.0  \n",
       "485079503025008 13  1        1      0      0      0       0       0                      0.0  \n",
       "485079503025017 17  1        1      0      0      0       0       0                      0.0  \n",
       "485079503025023 16  1        1      0      0      0       0       0                      0.0  \n",
       "\n",
       "[69959 rows x 7 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ab[df_ab.pct_male_2010 != df_ab.pct_male_2020]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "60215e04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0      69021\n",
       "100.0      938\n",
       "Name: pct_trans_2020, dtype: int64"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ab[df_ab.pct_male_2010 != df_ab.pct_male_2020].pct_trans_2020.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "2a2d0782",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0      69268\n",
       "100.0      691\n",
       "Name: pct_discordant_sex, dtype: int64"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ab[df_ab.pct_male_2010 != df_ab.pct_male_2020].pct_discordant_sex.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "15e195de",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9877213796652325"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ab[df_ab.pct_male_2010 != df_ab.pct_male_2020].pct_discordant_sex.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "4b008e81",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_scenario_2 = sum(\n",
    "    df_ab[df_ab.pct_male_2010 != df_ab.pct_male_2020].pct_discordant_sex == 100\n",
    "    )\n",
    "N_scenario_2 = sum(df_ab.pct_male_2010 != df_ab.pct_male_2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "c22ea0d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(691, 69959)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_scenario_2, N_scenario_2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22b534dc",
   "metadata": {},
   "source": [
    "# next a version with household swapping to protect against disclosure\n",
    "# I hypothesize that it is going to be just 10% lower"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "04f230f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "p_swap = 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "b731d81c",
   "metadata": {},
   "outputs": [],
   "source": [
    "locations_2010 = df.hh_id_2010.unique()\n",
    "\n",
    "random_location = np.random.choice(all_locations, size=len(locations_2010),  # induces distribution on geoid that is proportional to number of households\n",
    "                                   replace=True)\n",
    "\n",
    "reported_locations_2010 = np.where(np.random.uniform(size=len(locations_2010)) < p_swap,\n",
    "                                   random_location,\n",
    "                                   locations_2010,\n",
    "                                  )\n",
    "\n",
    "s_reported_location_2010 = pd.Series(reported_locations_2010,\n",
    "                                     index=locations_2010)\n",
    "\n",
    "df['reported_hh_id_2010'] = df.hh_id_2010.map(s_reported_location_2010)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "5227060d",
   "metadata": {},
   "outputs": [],
   "source": [
    "locations_2020 = df.hh_id_2020.unique()\n",
    "\n",
    "random_location = np.random.choice(all_locations, size=len(locations_2020),\n",
    "                                   replace=True)\n",
    "\n",
    "reported_locations_2020 = np.where(np.random.uniform(size=len(locations_2020)) < p_swap,\n",
    "                                   random_location,\n",
    "                                   locations_2020,\n",
    "                                  )\n",
    "\n",
    "s_reported_location_2020 = pd.Series(reported_locations_2020,\n",
    "                                     index=locations_2020)\n",
    "\n",
    "df['reported_hh_id_2020'] = df.hh_id_2020.map(s_reported_location_2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "082afad7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# number of trans families identified by age, geoid from reconstruction without noise\n",
    "\n",
    "df['reported_geoid_2010'] = df.reported_hh_id_2010.map(lambda x: int(x.split('-')[0]))\n",
    "df['reported_geoid_2020'] = df.reported_hh_id_2020.map(lambda x: int(x.split('-')[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "7414a3ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# without reidentification to link on, there is still a risk of identifying a block with a trans kid\n",
    "# by finding blocks where there was a single kids of a given age in 2010 and that age+10 in 2020 and\n",
    "# different reported sex\n",
    "df['reported_male_2010'] = (df.reported_sex_2010 == 1)\n",
    "df['geoid'] = df.reported_geoid_2010\n",
    "df['age'] = df.age_2010 + 10 # add ten to make merge easier\n",
    "g = df.groupby(['geoid', 'age',\n",
    "                'hispanic', 'racwht', 'racblk',\n",
    "                'racasn', 'racaian', 'racnhpi', 'racsor'])\n",
    "\n",
    "df_a = pd.DataFrame({'n_simulants': g.pweight.sum()})\n",
    "df_a['pct_male'] = 100*g.reported_male_2010.mean()\n",
    "df_a['pct_trans'] = 100*g.trans.mean()\n",
    "# df_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "d3a5f6c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1453203"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['reported_male_2020'] = (df.reported_sex_2020 == 1)\n",
    "df['geoid'] = df.reported_geoid_2020\n",
    "df['age'] = df.age_2020\n",
    "df['discordant_sex'] = (df.reported_sex_2010 != df.reported_sex_2020)\n",
    "g = df.groupby(['geoid', 'age',\n",
    "                'hispanic', 'racwht', 'racblk',\n",
    "                'racasn', 'racaian', 'racnhpi', 'racsor'])\n",
    "\n",
    "df_b = pd.DataFrame({'n_simulants': g.pweight.sum()})\n",
    "df_b['pct_male'] = 100*g.reported_male_2020.mean()\n",
    "df_b['pct_trans'] = 100*g.trans.mean()\n",
    "df_b['pct_discordant_sex'] = 100*g.discordant_sex.mean()\n",
    "n_unique_2020 = sum(df_b.n_simulants == 1)\n",
    "n_unique_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "dfc0c992",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ab = df_a[df_a.n_simulants==1].copy()\n",
    "df_ab.columns = ['n_simulants_2010', 'pct_male_2010', 'pct_trans_2010']\n",
    "\n",
    "df_ab['n_simulants_2020'] = df_b[df_b.n_simulants==1].n_simulants\n",
    "df_ab['pct_male_2020'] = df_b[df_b.n_simulants==1].pct_male\n",
    "df_ab['pct_trans_2020'] = df_b[df_b.n_simulants==1].pct_trans\n",
    "df_ab['pct_discordant_sex'] = df_b[df_b.n_simulants==1].pct_discordant_sex\n",
    "\n",
    "df_ab = df_ab.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "b6694a04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(657, 77109)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_scenario_3 = sum(\n",
    "    df_ab[df_ab.pct_male_2010 != df_ab.pct_male_2020].pct_discordant_sex == 100\n",
    "    )\n",
    "N_scenario_3 = sum(df_ab.pct_male_2010 != df_ab.pct_male_2020)\n",
    "n_scenario_3, N_scenario_3-n_scenario_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e2d4e830",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.00844842218964586"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_scenario_3 / N_scenario_3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "b875fefb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(691, 69268)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_scenario_2, N_scenario_2-n_scenario_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "68f9b08b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.009877213796652325"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_scenario_2 / N_scenario_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "e690030e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.92040520984081"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "100 * (n_scenario_2 - n_scenario_3) / n_scenario_2  # pct decrease from scenario 2 to 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e801f72b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pyomo_env",
   "language": "python",
   "name": "pyomo_env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}