update socsec and uncomp

docs/book/content/methods/CBO_Baseline_Updating_Instructions.md

@@ -96,9 +96,9 @@ Source: [OASI Trust Fund Annual Trustees Report](https://www.ssa.gov/oact/TR/)
 Table VI.C4. Operations of the OASI Trust Fund, Table VI.C4, Column:
 `Scheduled Benefits: Intermediate Level`
 
-Previous: [2019 Report](https://www.ssa.gov/oact/TR/2019/tr2019.pdf)
+Previous: [2023 Report](https://www.ssa.gov/oact/TR/2023/VI_C_SRfyproj.html)
 
-Current: [2020 Report](https://www.ssa.gov/oact/TR/2020/VI_C_SRfyproj.html)
+Current: [2023 Report](https://www.ssa.gov/oact/TR/2023/VI_C_SRfyproj.html)
 
 Projections are taken directly from the `Scheduled Benefits: Intermediate Level`
 column of this table.
@@ -108,7 +108,7 @@ column of this table.
 Source: [BLS Database](http://data.bls.gov/timeseries/CUSR0000SAM?output_view=pct_1mth)
 
 Series ID: CUSR0000SAM
-Access Date: February 16 2024
+Access Date: 
 
 Manual Instructions:
 
@@ -124,7 +124,7 @@ Manual Instructions:
 
 Source: [CBO Unemployment Compensation projections](https://www.cbo.gov/about/products/baseline-projections-selected-programs#24)
 
-Previous: [May 2022](https://www.cbo.gov/system/files/2022-05/51316-2022-05-unemployment.xlsx)
+Previous: [February 2024](https://www.cbo.gov/system/files/2024-02/51316-2024-02-unemployment.xlsx)
 
 Current: [February 2024](https://www.cbo.gov/system/files/2024-02/51316-2024-02-unemployment.xlsx)
 

puf_stage1/CBO_baseline.csv

@@ -10,7 +10,7 @@ RENTS,262.1,333.7,402.8,485.3,533.0,595.8,610.8,659.6,707.4,652.7,681.9,692.1,71
 CGNS,498.0,263.0,394.0,404.0,647.0,511.0,716.0,725.0,641.0,871.3,944.0,881.0,1147.7,2072.7,1319.1,1398.9,1313.9,1293.6,1264.1,1265.0,1285.4,1318.5,1359.7,1406.4,1457.2,1511.1,1567.4
 BOOK,1285.1,1397.0,1746.4,1816.6,2022.8,2106.9,2072.9,2088.1,2073.5,2128.9,2305.0,2367.8,2260.1,2771.1,2925.1,3215.7,3263.0,3353.8,3427.5,3512.3,3621.3,3748.0,3863.6,3984.2,4121.1,4262.4,4414.5
 RETS,142.5,140.5,142.9,145.4,144.9,146.6,148.3,147.8,149.7,149.9,152.5,153.5,160.1,164.4,163.2,165.2,167.1,168.7,170.3,171.9,173.6,175.3,177.0,178.6,180.4,182.1,183.8
-SOCSEC,503.0,544.5,572.5,591.5,627.2,663.2,698.2,733.7,762.1,791.1,833.0,888.1,940.2,982.7,1063.4,1148.7,1226.1,1302.8,1383.6,1467.4,1557.4,1650.6,1746.5,1844.8,,,
+SOCSEC,503.0,544.5,572.5,591.5,627.2,663.2,698.2,733.7,762.1,791.1,833.0,888.1,940.2,982.7,1063.4,1190.9,1291.4,1372.6,1454.4,1538.5,1627.6,1719.7,1813.9,1909.2,2005.2,2106.0,2211.9
 CPIU,215.3,214.5,218.1,224.9,229.6,233.0,236.7,237.0,240.0,245.1,251.1,255.6,258.8,271.0,292.9,304.7,312.6,320.4,327.9,335.1,342.4,350.0,357.8,365.8,374.1,382.5,391.1
 CPIM,364.1,375.6,388.4,400.3,414.9,425.1,435.3,446.8,463.7,475.3,484.7,498.4,518.9,525.3,546.5,549.1,556.8,533.8,541.3,548.5,555.8,563.4,571.2,579.2,587.5,595.9,604.5
-UCOMP,51.1,131.2,138.9,107.2,83.6,62.2,43.8,39.9,31.2,31.1,28.7,27.0,27.925,240.656,36.743,27.511,31.4,33.922,36.907,40.003,43.608,46.439,48.195,49.829,51.225,,
+UCOMP,51.1,131.2,138.9,107.2,83.6,62.2,43.8,39.9,31.2,31.1,28.7,27.0,27.925,240.656,36.743,27.511,36.8,42.5,44.0,46.0,48.0,50.2,52.4,54.4,56.1,57.7,59.4

puf_stage1/updatecbo.py

@@ -237,19 +237,23 @@ def update_socsec(url, baseline, text_args):
     r = session.get(url)
     # we can determine the latest year by looking at all of the years availeble
     # in the first drop down and adding one.
-    selector = r.html.find("select#yh1")[0]
-    latest_yr = max([int(yr) for yr in selector.text.split()]) + 1
+    # selector = r.html.find("select#yh1")[0]
+
+    # check https://www.ssa.gov/oact/TR/ for the latest year
+    latest_yr = 2023
+    #latest_yr = max([int(yr) for yr in selector.text.split()]) + 1
     report = f"{latest_yr} Report"
     if report == text_args["socsec_cur_report"]:
         print("\tNo new data since last update")
         return baseline, text_args
 
+
     socsec_url = f"https://www.ssa.gov/oact/TR/{latest_yr}/VI_C_SRfyproj.html"
     match_txt = "Operations of the OASI Trust Fund, Fiscal Years"
     html = pd.read_html(socsec_url, match=match_txt)[0]
     # merge the columns with years and data into one
     sub_data = pd.concat(
-        [html["Fiscal year", "Fiscal year.1"], html["Cost", "Sched-uled benefits"]],
+        [html["Fiscal year", "Fiscal year.1"], html["Cost", "Scheduled benefits"]],
         axis=1,
     )
     sub_data.columns = ["year", "cost"]
@@ -376,13 +380,18 @@ def update_ucomp(url, baseline, text_args):
     elif report == "February 2021":
         print("Latest data is from pandemic. Enter by hand")
         return baseline, text_args
-    data = pd.read_excel(ucomp_url, skiprows=3, index_col=0, thousands=",")
+    data = pd.read_excel(ucomp_url, skiprows=7, index_col=[0,1,2], thousands=",")
     try:
         benefits = data.loc["Budget Authority"].dropna().astype(int) / 1000
     except KeyError:
         benefits = data.loc["Budget Authority"].dropna().astype(int) / 1000
     benefits = benefits.round(1)
     df = pd.DataFrame(benefits).transpose()
+    # drop items whose index are not years
+    for indx in df.index:
+        if type(indx) != int:
+            df = df.drop(indx)
+    df = pd.DataFrame(df.values, index=df.index).transpose()
     df.index = ["UCOMP"]
     df.columns = df.columns.astype(str)
     baseline.update(df)