diff --git a/README.md b/README.md index 0b3467b..15501ec 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ direction TB inputs_controls_jobs { run_id INT UK, FK year INT UK + ownership_title NVARCHAR(50) UK industry_code NVARCHAR(5) UK metric NVARCHAR(4) UK value INT @@ -209,6 +210,7 @@ direction TB run_id INT UK, FK year INT UK mgra INT UK, FK + ownership_title NVARCHAR(50) UK industry_code NVARCHAR(5) UK value INT } diff --git a/python/employment.py b/python/employment.py index 29becf4..1cdd0a9 100644 --- a/python/employment.py +++ b/python/employment.py @@ -84,7 +84,7 @@ def _get_lodes_data(year: int) -> pd.DataFrame: ), ], ignore_index=True, - )[["year", "block", "industry_code", "jobs"]] + )[["year", "block", "ownership_title", "industry_code", "jobs"]] return combined_data @@ -97,18 +97,18 @@ def _aggregate_lodes_to_mgra( This function allocates jobs from Census blocks to MGRAs using distributions from the California Employment Development Department (EDD) point-level dataset. Blocks with no EDD data available use a simple land area intersection to allocate jobs to - MGRAs. The allocation first attempts to allocate within industry codes using EDD - data, then falls back to using EDD data without considering industry codes, and - finally falls back to using the land area intersection. + MGRAs. The allocation first attempts to allocate within SANDAG employment categories + using EDD data, then falls back to using EDD data without considering categories, + and finally falls back to using the land area intersection. Args: combined_data: LODES data with columns: year, block, industry_code, jobs - xref: Crosswalk with columns: block, mgra, pct_industry, pct_edd, pct_area, flag + xref: Crosswalk with columns: block, mgra, pct_edd_category, pct_edd, pct_area, flag year: The year for which to aggregate data Returns: Aggregated data at MGRA level with columns: run_id, year, mgra, - industry_code, value + ownership_title, industry_code, value """ # Get MGRA data from SQL with utils.ESTIMATES_ENGINE.connect() as con: @@ -123,118 +123,44 @@ def _aggregate_lodes_to_mgra( params={"run_id": utils.RUN_ID}, ) - # Get unique industry codes and cross join with MGRA data - unique_industries = combined_data["industry_code"].unique() jobs = ( - mgra_data.merge(pd.DataFrame({"industry_code": unique_industries}), how="cross") + # Get unique SANDAG employment categories and cross join with MGRA data + mgra_data.merge( + combined_data[["ownership_title", "industry_code"]] + .drop_duplicates() + .reset_index(drop=True), + how="cross", + ) .assign(year=year) + # Get the LODES data and allocated to MGRAs using the crosswalk and allocation percentages .merge( - combined_data.merge(xref, on=["block", "industry_code"], how="inner") + combined_data.merge( + xref, on=["block", "ownership_title", "industry_code"], how="inner" + ) .assign( value=lambda df: df["jobs"] * np.where( - df["flag"] == "pct_industry", - df["pct_industry"], + df["flag"] == "pct_edd_category", + df["pct_edd_category"], np.where(df["flag"] == "pct_edd", df["pct_edd"], df["pct_area"]), ) ) - .groupby(["year", "mgra", "industry_code"], as_index=False)["value"] + .groupby( + ["year", "mgra", "ownership_title", "industry_code"], as_index=False + )["value"] .sum(), - on=["year", "mgra", "industry_code"], + on=["year", "mgra", "ownership_title", "industry_code"], how="left", ) .fillna({"value": 0}) .assign(run_id=utils.RUN_ID)[ - ["run_id", "year", "mgra", "industry_code", "value"] + ["run_id", "year", "mgra", "ownership_title", "industry_code", "value"] ] ) return jobs -def _distribute_self_emp_to_mgra( - b24080: pd.DataFrame, xref: pd.DataFrame -) -> pd.DataFrame: - """Distribute subregional self-employment counts to MGRA level. - - This function allocates self-employment counts from block groups (for years - post 2012) or tracts (for 2010-2012) to MGRAs using allocation percentages. - The allocation percentages are based on the distribution of persons aged - 18-64, total population, or an equal split across intersection MGRAs - depending on data availability within the block group or tract. - - Args: - b24080: DataFrame containing subregional self-employment counts. - Must include columns (year, geography, industry_code, value). - xref: Geography crosswalk DataFrame with columns - (geography, mgra, flag, pct_18_64, pct_pop, pct_split) - - Returns: - Self employment counts at the MGRA level - """ - # Check that required columns are present - required_b24080_cols = {"year", "geography", "industry_code", "value"} - required_xref_cols = { - "geography", - "mgra", - "flag", - "pct_18_64", - "pct_pop", - "pct_split", - } - if not required_b24080_cols.issubset(b24080.columns): - raise ValueError( - f"B24080 DataFrame is missing required columns: {required_b24080_cols - set(b24080.columns)}" - ) - if not required_xref_cols.issubset(xref.columns): - raise ValueError( - f"xref DataFrame is missing required columns: {required_xref_cols - set(xref.columns)}" - ) - - # Check that flag column only contains expected values - expected_flags = {"pct_18_64", "pct_pop", "pct_split"} - if not set(xref["flag"].unique()).issubset(expected_flags): - raise ValueError( - f"xref 'flag' column contains unexpected values: {set(xref['flag'].unique()) - expected_flags}" - ) - - # Merge subregional self employment counts with MGRA crosswalk - merged = b24080.merge(xref, on="geography", how="inner") - - # Calculate weighted value based on flag - merged = merged.assign( - weighted_value=np.select( - [ - merged["flag"] == "pct_18_64", - merged["flag"] == "pct_pop", - merged["flag"] == "pct_split", - ], - [ - merged["value"] * merged["pct_18_64"], - merged["value"] * merged["pct_pop"], - merged["value"] * merged["pct_split"], - ], - default=np.nan, - ) - ) - - if merged["weighted_value"].isna().any(): - raise ValueError( - "Unexpected allocation flag found; expected one of {'pct_18_64', 'pct_pop', 'pct_split'}" - ) - - # Sum weighted values to the MGRA level - merged = ( - merged.groupby(["year", "mgra", "industry_code"])["weighted_value"] - .sum() - .reset_index() - .assign(run_id=utils.RUN_ID) - .rename(columns={"weighted_value": "value"}) - )[["run_id", "year", "mgra", "industry_code", "value"]] - - return merged - - def _get_jobs_inputs(year: int) -> dict[str, pd.DataFrame]: """Get input data related to jobs for a specified year. @@ -257,51 +183,13 @@ def _get_jobs_inputs(year: int) -> dict[str, pd.DataFrame]: params={ "year": year, }, - ) - - # Get self-employed totals and append to control_totals - with open(utils.SQL_FOLDER / "employment/get_region_self_emp.sql") as file: - self_emp_control = utils.read_sql_query_fallback( - sql=sql.text(file.read()), - con=con, - params={ - "year": year, - }, - ) - - jobs_inputs["control_totals"] = pd.concat( - [jobs_inputs["control_totals"], self_emp_control], - ignore_index=True, - ) - - jobs_inputs["control_totals"]["run_id"] = utils.RUN_ID - - # Get self-employed block group data - with open(utils.SQL_FOLDER / "employment/get_B24080.sql") as file: - jobs_inputs["B24080"] = utils.read_sql_query_fallback( - sql=sql.text(file.read()), - con=con, - params={ - "year": year, - }, - ) - - # Get census block group or tract to MGRA crosswalk - with open(utils.SQL_FOLDER / "employment/xref_se_to_mgra.sql") as file: - jobs_inputs["xref_se_to_mgra"] = pd.read_sql_query( - sql=sql.text(file.read()), - con=con, - params={ - "run_id": utils.RUN_ID, - "year": year, - }, - ) + ).assign(run_id=utils.RUN_ID) with utils.GIS_ENGINE.connect() as con: # Get crosswalk from Census blocks to MGRAs with open(utils.SQL_FOLDER / "employment/xref_block_to_mgra.sql") as file: jobs_inputs["xref_block_to_mgra"] = utils.read_sql_query_fallback( - max_lookback=2, + max_lookback=1, sql=sql.text(file.read()), con=con, params={ @@ -324,11 +212,12 @@ def _get_jobs_inputs(year: int) -> dict[str, pd.DataFrame]: military_control_totals = ( jobs_inputs["military_emp"] - .groupby(["run_id", "year", "industry_code", "metric"], as_index=False)[ - "value" - ] + .groupby( + ["run_id", "year", "ownership_title", "industry_code", "metric"], + as_index=False, + )["value"] .sum() - )[["run_id", "year", "industry_code", "metric", "value"]] + )[["run_id", "year", "ownership_title", "industry_code", "metric", "value"]] jobs_inputs["control_totals"] = pd.concat( [jobs_inputs["control_totals"], military_control_totals], @@ -348,14 +237,6 @@ def _validate_jobs_inputs(jobs_inputs: dict[str, pd.DataFrame]) -> None: negative={}, null={}, ) - # Self Employed only includes block groups with self-employed individuals therefore - # no row count validation performed - tests.validate_data( - "Self-employed block group data", - jobs_inputs["B24080"], - negative={}, - null={}, - ) # No row count validation performed as xref is many-to-many # NULLs are allowed in the result set tests.validate_data( @@ -363,13 +244,6 @@ def _validate_jobs_inputs(jobs_inputs: dict[str, pd.DataFrame]) -> None: jobs_inputs["xref_block_to_mgra"], negative={}, ) - # No row count validation performed as xref is many-to-many - tests.validate_data( - "xref_se_to_mgra", - jobs_inputs["xref_se_to_mgra"], - negative={}, - null={}, - ) tests.validate_data( "Military employment data", jobs_inputs["military_emp"], @@ -380,7 +254,7 @@ def _validate_jobs_inputs(jobs_inputs: dict[str, pd.DataFrame]) -> None: tests.validate_data( "Jobs control totals", jobs_inputs["control_totals"], - row_count={"key_columns": {"industry_code"}}, + row_count={"key_columns": {("ownership_title", "industry_code")}}, negative={}, null={}, ) @@ -397,52 +271,64 @@ def _create_jobs_output( Returns: Controlled employment data. """ - # Create MGRA level jobs data by combining LODES and self-employment data + # Create MGRA level jobs data by combining LODES and military data mgra_jobs = pd.concat( [ # Aggregate LODES jobs to MGRA level _aggregate_lodes_to_mgra( jobs_inputs["lodes_data"], jobs_inputs["xref_block_to_mgra"], year ), - # Distribute self-employment data to MGRA level - _distribute_self_emp_to_mgra( - jobs_inputs["B24080"], jobs_inputs["xref_se_to_mgra"] - ), # Include military employment at MGRA level jobs_inputs["military_emp"][ - ["run_id", "year", "mgra", "industry_code", "value"] + ["run_id", "year", "mgra", "ownership_title", "industry_code", "value"] ], ], ignore_index=True, - ).sort_values(by=["mgra", "industry_code"]) + ).sort_values(by=["mgra", "ownership_title", "industry_code"]) # Create list to store controlled values for each industry results = [] - # Apply integerize_1d to each industry_code - for industry_code in mgra_jobs["industry_code"].unique(): - # Filter for this industry_code - naics_mask = mgra_jobs.loc[mgra_jobs["industry_code"] == industry_code] - - # Get control value and apply integerize_1d - control_value = ( - jobs_inputs["control_totals"] - .loc[ - jobs_inputs["control_totals"]["industry_code"] == industry_code, "value" + # Apply integerize_1d to each SANDAG employment category + for ownership_title in mgra_jobs["ownership_title"].unique(): + for industry_code in mgra_jobs["industry_code"].unique(): + # Filter for this ownership_title and industry_code + mask = mgra_jobs.loc[ + (mgra_jobs["ownership_title"] == ownership_title) + & (mgra_jobs["industry_code"] == industry_code) ] - .iloc[0] - ) - results.append( - naics_mask.assign( - value=utils.integerize_1d( - data=naics_mask["value"], - control=control_value, - methodology="weighted_random", - generator=generator, + # If no records are returned, skip to next iteration + if mask.empty: + continue + else: + # Get control value and apply integerize_1d + control_value = ( + jobs_inputs["control_totals"] + .loc[ + ( + jobs_inputs["control_totals"]["ownership_title"] + == ownership_title + ) + & ( + jobs_inputs["control_totals"]["industry_code"] + == industry_code + ), + "value", + ] + .iloc[0] + ) + + results.append( + mask.assign( + value=utils.integerize_1d( + data=mask["value"], + control=control_value, + methodology="weighted_random", + generator=generator, + ) + ) ) - ) - ) return {"results": pd.concat(results, ignore_index=True)} @@ -452,7 +338,7 @@ def _validate_jobs_outputs(jobs_outputs: dict[str, pd.DataFrame]) -> None: tests.validate_data( "Controlled jobs data", jobs_outputs["results"], - row_count={"key_columns": {"mgra", "industry_code"}}, + row_count={"key_columns": {"mgra", ("ownership_title", "industry_code")}}, negative={}, null={}, ) diff --git a/python/tests.py b/python/tests.py index d1ac869..49778af 100644 --- a/python/tests.py +++ b/python/tests.py @@ -37,12 +37,9 @@ 2020: 736, }, "jurisdiction": 19, - # The industry_code is a variable to group employment data into. Almost all - # codes are 2-digit naics codes. The 2-digit naics code 72 was split into 721 - # and 722 3-digit naics code. Self employment and military active duty data do - # not natively have a naics code so it is therefore assigned to 'SE' and 'MIL' - # respectively, as the naics codes are being treated as strings. - "industry_code": 23, + # See https://github.com/SANDAG/Estimates-Program/issues/281 + # For exhaustive list of SANDAG employment categories + Military + ("ownership_title", "industry_code"): 24, }, "series": { 15: { @@ -169,14 +166,18 @@ def validate_data(table_name: str, data: pd.DataFrame, **kwargs) -> None: def _validate_row_count( - table_name: str, data: pd.DataFrame, key_columns: set[str], year: int = None + table_name: str, + data: pd.DataFrame, + key_columns: set, + year: int = None, ) -> None: """Verify that the provided data has the correct number of rows The correct number of rows is determined by the input 'key_columns', under the assumption that input data has functionally the SQL CROSS JOIN of 'key_columns'. The number of values in each key column is determined by the variable above labeled - '_DISTINCT_COUNTS' + '_DISTINCT_COUNTS'. We do allow for tuples to be passed as key columns with distinct + counts assigned to the tuple. Args: table_name: The name of the table. The only purpose of this is to make error @@ -200,10 +201,20 @@ def _validate_row_count( # Verify that the provided key columns actually exist and we have data for them for column in key_columns: - if column not in data.columns: - raise ValueError( - f"'{table_name}' is missing the required key column '{column}'" - ) + # If key column provided as a tuple, check each individual column in the tuple is in the dataset + if isinstance(column, tuple): + for sub_column in column: + if sub_column not in data.columns: + raise ValueError( + f"'{table_name}' is missing the required key column " + f"'{sub_column}'" + ) + else: + if column not in data.columns: + raise ValueError( + f"'{table_name}' is missing the required key column '{column}'" + ) + # Both strings and tuples are allowed in the _DISTINCT_COUNTS dictionary, so we can check for both if column not in _DISTINCT_COUNTS.keys(): raise ValueError( f"'tests.py' is missing data for the key column '{column}'. Fill in the " @@ -222,8 +233,9 @@ def _validate_row_count( else: unique_key_values[column] = _DISTINCT_COUNTS[column] - # Check that the total number of rows is correct, assuming that we do the CROSS JOIN - # of all keys columns + # For key columns defined as tuples, we consider these as combined columns + # with a single unique value count when calculating the total number of rows + # Check that the total number of rows is correct, using the CROSS JOIN of all key columns n_rows = math.prod(unique_key_values.values()) if data.shape[0] != n_rows: row_count_explanation = " x ".join( diff --git a/sql/create_objects.sql b/sql/create_objects.sql index 4a59332..424e0a0 100644 --- a/sql/create_objects.sql +++ b/sql/create_objects.sql @@ -38,11 +38,12 @@ GO CREATE TABLE [inputs].[controls_jobs] ( [run_id] INT NOT NULL, [year] INT NOT NULL, + [ownership_title] NVARCHAR (50) NOT NULL, [industry_code] NVARCHAR(5) NOT NULL, [metric] NVARCHAR(4) NOT NULL, [value] INT NOT NULL, INDEX [ccsi_inputs_controls_jobs] CLUSTERED COLUMNSTORE, - CONSTRAINT [ixuq_inputs_controls_jobs] UNIQUE ([run_id], [year], [industry_code], [metric]) WITH (DATA_COMPRESSION = PAGE), + CONSTRAINT [ixuq_inputs_controls_jobs] UNIQUE ([run_id], [year], [ownership_title], [industry_code], [metric]) WITH (DATA_COMPRESSION = PAGE), CONSTRAINT [fk_inputs_controls_jobs_run_id] FOREIGN KEY ([run_id]) REFERENCES [metadata].[run] ([run_id]), CONSTRAINT [chk_non_negative_inputs_controls_jobs] CHECK ([value] >= 0) ) @@ -306,11 +307,12 @@ GO CREATE TABLE [outputs].[jobs] ( [run_id] INT NOT NULL, [year] INT NOT NULL, - [mgra] INT NOT NULL, + [mgra] INT NOT NULL, + [ownership_title] NVARCHAR (50) NOT NULL, [industry_code] NVARCHAR(5) NOT NULL, [value] INT NOT NULL, INDEX [ccsi_outputs_jobs] CLUSTERED COLUMNSTORE, - CONSTRAINT [ixuq_outputs_jobs] UNIQUE ([run_id], [year], [mgra], [industry_code]) WITH (DATA_COMPRESSION = PAGE), + CONSTRAINT [ixuq_outputs_jobs] UNIQUE ([run_id], [year], [mgra], [ownership_title], [industry_code]) WITH (DATA_COMPRESSION = PAGE), CONSTRAINT [fk_outputs_jobs_run_id] FOREIGN KEY ([run_id]) REFERENCES [metadata].[run] ([run_id]), CONSTRAINT [fk_outputs_jobs_mgra] FOREIGN KEY ([run_id], [mgra]) REFERENCES [inputs].[mgra] ([run_id], [mgra]), CONSTRAINT [chk_non_negative_outputs_jobs] CHECK ([value] >= 0) diff --git a/sql/employment/get_B24080.sql b/sql/employment/get_B24080.sql deleted file mode 100644 index 2e31ca0..0000000 --- a/sql/employment/get_B24080.sql +++ /dev/null @@ -1,51 +0,0 @@ --- Get ACS 5-year self-employment counts by census blockgroup (2013+) or tract (2010-2012) - --- Initialize parameters ----------------------------------------------------- -DECLARE @year INTEGER = :year; -DECLARE @msg nvarchar(31) = 'ACS 5-Year Table does not exist'; - --- Send error message if no data exists -------------------------------------- -IF NOT EXISTS ( - SELECT TOP (1) * - FROM [acs].[detailed].[tables] - WHERE - [name] = 'B24080' - AND [year] = @year - AND [product] = '5Y' -) -SELECT @msg AS [msg] -ELSE -BEGIN - - -- Get the ACS data ---------------------------------------------------------- - SELECT - @year AS [year], - CASE - WHEN @year BETWEEN 2010 AND 2012 THEN [tract] - ELSE [blockgroup] - END AS [geography], - 'SE' AS [industry_code], - SUM([value]) AS [value] - FROM [acs].[detailed].[values] - INNER JOIN [acs].[detailed].[geography] - ON [values].[geography_id] = [geography].[geography_id] - INNER JOIN [acs].[detailed].[variables] - ON [values].[table_id] = [variables].[table_id] - AND [variables].[variable] = [values].[variable] - INNER JOIN [acs].[detailed].[tables] - ON [values].[table_id] = [tables].[table_id] - WHERE - [tables].[name] = 'B24080' - AND [tables].[product] = '5Y' - AND REPLACE([variables].[label], ':', '') IN ( - 'Estimate!!Total!!Male!!Self-employed in own not incorporated business workers', - 'Estimate!!Total!!Female!!Self-employed in own not incorporated business workers' - ) - AND [tables].[year] = @year - GROUP BY - CASE - WHEN @year BETWEEN 2010 AND 2012 THEN [tract] - ELSE [blockgroup] - END - ORDER BY [geography] -END \ No newline at end of file diff --git a/sql/employment/get_lodes_data.sql b/sql/employment/get_lodes_data.sql index 5f9769d..037b62c 100644 --- a/sql/employment/get_lodes_data.sql +++ b/sql/employment/get_lodes_data.sql @@ -1,18 +1,25 @@ /* -Get LEHD LODES data. + Get LEHD LODES data mapped to SANDAG employment categories. -The mapping below used for [CNS01] to [CNS20] to [niacs_code] (2-digit NAICS) in WAC -section of the document linked below. The mapping for [SEG] and [TYPE] are included in -the OD section of document linked below. + The mapping below used for [CNS01] to [CNS20] to [niacs_code] (2-digit NAICS) in WAC + section of the document linked below. The mapping for [SEG] and [TYPE] are included in + the OD section of document linked below. + https://lehd.ces.census.gov/doc/help/onthemap/LODESTechDoc.pdf + For any other LEHD LODES data questions check: https://lehd.ces.census.gov/data/ -https://lehd.ces.census.gov/doc/help/onthemap/LODESTechDoc.pdf + SANDAG employment categories are defined here: + https://github.com/SANDAG/Estimates-Program/issues/281 -For any other LEHD LODES data questions check: https://lehd.ces.census.gov/data/ + This result set cannot be used to determine raw employment counts or aggregated + across ownerships as data is purposefully duplicated with the State and Local + Government categories due to limitations of the LEHD LODES data. */ +SET NOCOUNT ON; -- Initialize parameters ----------------------------------------------------- -DECLARE @year integer = :year; -DECLARE @msg nvarchar(25) = 'LODES data does not exist'; +DECLARE @year INTEGER = :year; +DECLARE @msg NVARCHAR(25) = 'LODES data does not exist'; +DECLARE @lodes_version INTEGER = 2 -- lodes v8.4 -- Send error message if no data exists -------------------------------------- IF NOT EXISTS ( @@ -21,7 +28,7 @@ IF NOT EXISTS ( WHERE [SEG] = 'S000' -- 'S000' = 'Total number of jobs' AND [TYPE] = 'JT00' -- 'JT00' = 'All Jobs' - AND [version] = 2 -- lodes v8.4 + AND [version] = @lodes_version AND [YEAR] = @year ) BEGIN @@ -29,60 +36,186 @@ BEGIN END ELSE BEGIN - -- Build the return table of QCEW control Totals by industry_code (NAICS) --- - SELECT - [YEAR] AS [year], - -- https://github.com/SANDAG/Estimates-Program/issues/193 - CASE - WHEN [w_geocode] = '060730106012030' THEN '060730106012027' - WHEN [w_geocode] IN ('060730183012003', '060730183012004') - THEN '060730183012010' - ELSE [w_geocode] - END AS [block], - [industry_code], - SUM([value]) AS [jobs] - FROM [socioec_data].[lehd].[lodes_8_wac] - CROSS APPLY ( - VALUES - ('11', [CNS01]), - ('21', [CNS02]), - ('22', [CNS03]), - ('23', [CNS04]), - ('31-33',[CNS05]), - ('42', [CNS06]), - ('44-45',[CNS07]), - ('48-49',[CNS08]), - ('51', [CNS09]), - ('52', [CNS10]), - ('53', [CNS11]), - ('54', [CNS12]), - ('55', [CNS13]), - ('56', [CNS14]), - ('61', [CNS15]), - ('62', [CNS16]), - ('71', [CNS17]), - ('72', [CNS18]), - ('81', [CNS19]), - ('92', [CNS20]) - ) AS u([industry_code], [value]) - WHERE - [SEG] = 'S000' -- 'S000' = 'Total number of jobs' - AND [TYPE] = 'JT00' -- 'JT00' = 'All Jobs' - AND [version] = 2 -- lodes v8.4 - AND [YEAR] = @year - GROUP BY - [YEAR], - -- https://github.com/SANDAG/Estimates-Program/issues/193 - CASE - WHEN [w_geocode] = '060730106012030' THEN '060730106012027' - WHEN [w_geocode] IN ('060730183012003', '060730183012004') - THEN '060730183012010' - ELSE [w_geocode] - END, - [industry_code] - ORDER BY - [year], - [block], - [industry_code] -END ------------------------------------------------------------------------------- \ No newline at end of file + -- Get LEHD LODES jobs by ownership and industry code (NAICS) ------------ + WITH [lodes_data] AS ( + -- Build the return table of QCEW control Totals by industry_code (NAICS) --- + SELECT + -- https://github.com/SANDAG/Estimates-Program/issues/193 + CASE + WHEN [w_geocode] = '060730106012030' THEN '060730106012027' + WHEN [w_geocode] IN ('060730183012003', '060730183012004') + THEN '060730183012010' + ELSE [w_geocode] + END AS [block], + CASE + WHEN [TYPE] = 'JT00' THEN 'Total Covered' + WHEN [TYPE] = 'JT02' THEN 'Private' + WHEN [TYPE] = 'JT04' THEN 'Federal Government' + ELSE NULL + END AS [ownership_title], + [industry_code], + SUM([value]) AS [jobs] + FROM [socioec_data].[lehd].[lodes_8_wac] + CROSS APPLY ( + VALUES + ('11', [CNS01]), + ('21', [CNS02]), + ('22', [CNS03]), + ('23', [CNS04]), + ('31-33',[CNS05]), + ('42', [CNS06]), + ('44-45',[CNS07]), + ('48-49',[CNS08]), + ('51', [CNS09]), + ('52', [CNS10]), + ('53', [CNS11]), + ('54', [CNS12]), + ('55', [CNS13]), + ('56', [CNS14]), + ('61', [CNS15]), + ('62', [CNS16]), + ('71', [CNS17]), + ('72', [CNS18]), -- LEHD LODES does not allow a 721/722 split + ('81', [CNS19]), + ('92', [CNS20]) + ) AS u([industry_code], [value]) + WHERE + [SEG] = 'S000' -- 'S000' = 'Total number of jobs' + AND [TYPE] IN ('JT00', 'JT02', 'JT04') + AND [version] = @lodes_version + AND [YEAR] = @year + GROUP BY + -- https://github.com/SANDAG/Estimates-Program/issues/193 + CASE + WHEN [w_geocode] = '060730106012030' THEN '060730106012027' + WHEN [w_geocode] IN ('060730183012003', '060730183012004') + THEN '060730183012010' + ELSE [w_geocode] + END, + CASE + WHEN [TYPE] = 'JT00' THEN 'Total Covered' + WHEN [TYPE] = 'JT02' THEN 'Private' + WHEN [TYPE] = 'JT04' THEN 'Federal Government' + ELSE NULL + END, + [industry_code] + ), + -- Private-only SANDAG employment categories ----------------------------- + [private] AS ( + SELECT + [block], + [ownership_title], + [industry_code], + [jobs] + FROM [lodes_data] + WHERE + [ownership_title] = 'Private' + AND [industry_code] IN ( + '11', + '21', + '22', + '23', + '31-33', + '42', + '44-45', + '48-49', + '51', + '52', + '53', + '54', + '55', + '56', + '81' + ) + ), + -- All Ownership SANDAG employment categories ---------------------------- + [total_covered] AS ( + SELECT + [block], + [ownership_title], + [industry_code], + [jobs] + FROM [lodes_data] + WHERE + [ownership_title] = 'Total Covered' + AND [industry_code] IN ( + '61', + '62', + '71', + '72' -- LEHD LODES does not allow a 721/722 split + ) + ), + -- Federal Government SANDAG employment categories ----------------------- + [federal_government] AS ( + SELECT + [block], + [ownership_title], + 'GOV' AS [industry_code], + SUM([jobs]) AS [jobs] + FROM [lodes_data] + WHERE + [ownership_title] = 'Federal Government' + -- Remove Total Covered SANDAG employment categories + AND [industry_code] NOT IN ( + '61', + '62', + '71', + '72' -- LEHD LODES does not allow a 721/722 split + ) + GROUP BY + [block], + [ownership_title] + ), + -- State and Local Government SANDAG employment categories --------------- + -- LEHD LODES only differentiates between Private and Federal + -- So we will use Total Covered minus Private and Federal + -- For both the "State Government" and "Local Government" + [state_local_government] AS ( + SELECT + [tt_total].[block], + [tt_total].[jobs] - ISNULL([tt_federal].[jobs], 0) AS [jobs] + FROM ( + SELECT + [block], + SUM([jobs]) AS [jobs] + FROM [lodes_data] + WHERE + [ownership_title] = 'Total Covered' + AND [industry_code] NOT IN ( + '61', + '62', + '71', + '72' -- LEHD LODES does not allow a 721/722 split + ) + GROUP BY [block] + ) AS [tt_total] + LEFT OUTER JOIN ( + SELECT + [block], + SUM([jobs]) AS [jobs] + FROM [lodes_data] + WHERE + [ownership_title] IN ('Federal Government', 'Private') + AND [industry_code] NOT IN ( + '61', + '62', + '71', + '72' -- LEHD LODES does not allow a 721/722 split + ) + GROUP BY [block] + ) AS [tt_federal] + ON [tt_total].[block] = [tt_federal].[block] + ) + -- Combine all ownership categories together and return result set ------- + SELECT @year AS [year], [block], [ownership_title], [industry_code], [jobs] FROM [private] + UNION ALL + SELECT @year AS [year], [block], [ownership_title], [industry_code], [jobs] FROM [total_covered] + UNION ALL + SELECT @year AS [year], [block], [ownership_title], 'GOV' AS [industry_code], [jobs] FROM [federal_government] + UNION ALL + SELECT @year AS [year], [block], 'State Government' AS [ownership_title], 'GOV' AS [industry_code], [jobs] FROM [state_local_government] + UNION ALL + SELECT @year AS [year], [block], 'Local Government' AS [ownership_title], 'GOV' AS [industry_code], [jobs] FROM [state_local_government] + ORDER BY [block], [industry_code] + +END \ No newline at end of file diff --git a/sql/employment/get_military_employment.sql b/sql/employment/get_military_employment.sql index e5f4caa..e4e8b75 100644 --- a/sql/employment/get_military_employment.sql +++ b/sql/employment/get_military_employment.sql @@ -1,10 +1,10 @@ /* -This query grabs the Military Active Duty (Job) Data and assigns counts to MGRA15. -This will assign 0 to MGRAs where there is no Military Jobs + This query grabs the Military Active Duty (Job) Data and assigns counts to MGRA15. + This will assign 0s to MGRAs where there are no military jobs. -Notes: - 1) This is assuming a connection to the GIS server - 2) currently only works using MGRA15 + Notes: + 1) This is assuming a connection to the GIS server + 2) currently only works using MGRA15 */ -- Initialize parameters ----------------------------------------------------- @@ -36,6 +36,7 @@ BEGIN @run_id AS [run_id], @year AS [year], [mgra], + 'Federal Government' AS [ownership_title], 'MIL' AS [industry_code], 'jobs' AS [metric], COALESCE(SUM([site_active_duty]), 0) AS [value] diff --git a/sql/employment/get_naics72_split.sql b/sql/employment/get_naics72_split.sql index 0697107..94cc61c 100644 --- a/sql/employment/get_naics72_split.sql +++ b/sql/employment/get_naics72_split.sql @@ -1,17 +1,17 @@ /* -This query provides a split of 2-digit NAICS 72 into 3-digit NAICS codes. -Point-level data is gotten from the confidential EDD dataset, assigned to -2020 Census Blocks and the percentage split of 2-digit NAICS 72 into the -3-digit NAICS codes of 721 and 722 is calculated within each block. + This query provides a split of 2-digit NAICS 72 into 3-digit NAICS codes. + Point-level data is gotten from the confidential EDD dataset, assigned to + 2020 Census Blocks and the percentage split of 2-digit NAICS 72 into the + 3-digit NAICS codes of 721 and 722 is calculated within each block. -Notes: - 1) This query assumes the connection is to the GIS server. - 2) Data prior to year 2017 is not present in the EDD view and must be - queried directly from the source database table. Note there is no 2016 - data available. - 3) If no split is present for a block, the regional percentage split is - substituted. All 2020 Census blocks are represented, except three - water-only slivers see: https://github.com/SANDAG/Estimates-Program/issues/193 + Notes: + 1) This query assumes the connection is to the GIS server. + 2) Data prior to year 2017 is not present in the EDD view and must be + queried directly from the source database table. Note there is no 2016 + data available. + 3) If no split is present for a block, the regional percentage split is + substituted. All 2020 Census blocks are represented, except three + water-only slivers see: https://github.com/SANDAG/Estimates-Program/issues/193 */ SET NOCOUNT ON; diff --git a/sql/employment/get_region_qcew.sql b/sql/employment/get_region_qcew.sql index 549086c..7252256 100644 --- a/sql/employment/get_region_qcew.sql +++ b/sql/employment/get_region_qcew.sql @@ -1,48 +1,104 @@ --- Initialize parameters ----------------------------------------------------- -DECLARE @year integer = :year; -DECLARE @msg nvarchar(25) = 'QCEW data does not exist'; - --- Send error message if no data exists -------------------------------------- -IF NOT EXISTS ( - SELECT TOP (1) * - FROM [socioec_data].[bls].[qcew_by_area_annual] - WHERE [year] = @year -) -SELECT @msg AS [msg] -ELSE -BEGIN - SELECT - [year], - [industry_code], - 'jobs' AS [metric], - SUM([annual_avg_emplvl]) AS [value] - FROM [socioec_data].[bls].[qcew_by_area_annual] - INNER JOIN [socioec_data].[bls].[industry_code] - ON [qcew_by_area_annual].[naics_id] = [industry_code].[naics_id] - WHERE [area_fips] = '06073' - AND [year] = @year - AND [industry_code] IN ( - '11', - '21', - '22', - '23', - '31-33', - '42', - '44-45', - '48-49', - '51', - '52', - '53', - '54', - '55', - '56', - '61', - '62', - '71', - '721', - '722', - '81', - '92' - ) - GROUP BY [year], [industry_code] -END \ No newline at end of file +/* + This SQL query calculates annual employment averages for SANDAG employment + categories from the BLS QCEW for a given year. SANDAG employment categories + are built from a combination of ownership and 2-digit industry codes, + excepting for the exclusion of unclassified NAICS 99 and the split of NAICS 72 + into 721 and 722. SANDAG employment categories are as follows: + + Total Covered - 61,62,71,721,722 + Private - 11,21,22,23,31-33,42,44-45,48-49,51,52,53,54,55,56,81 + Federal/State/Local Government - see Private industries above + NAICS 92 + + For SANDAG employment categories not directly derived from published + BLS QCEW annual averages we use summations of monthly employment totals + published quarterly to aggregate into SANDAG employment categories and perform + averaging and integerization at the final reporting step per guidance received + from BLS QCEW staff. See https://github.com/SANDAG/BLS/issues/55. + There are all the SANDAG employment categories that combine ownership + categories. + + Total Covered - 61,62,71,721,722 + Federal/State/Local Government - see Private industries above + NAICS 92 + + For SANDAG employment categories that are directly derived from published BLS + QCEW annual averages we use those numbers directly as they are calculated by + the BLS using unreleased microdata that is more accurate than the rounded + quarterly monthly data. These are all the "Private" ownership only categories. + + Private - 11,21,22,23,31-33,42,44-45,48-49,51,52,53,54,55,56,81 + + This is able to be done for years 2022-2025 as no suppression exists at the + aggregation levels required to create SANDAG employment categories. Data prior + to 2022 requires controlling at higher aggregation levels to fill in + gaps created by data suppression. +*/ + +SET NOCOUNT ON; +-- Initialize parameters and return table ------------------------------------ +-- Set year of BLS QCEW data to create SANDAG employment categories +DECLARE @year INTEGER = :year; + +-- Data suppression limits this query to 2022-2025 only +IF @year < 2022 OR @year > 2025 + THROW 50000, 'Data suppression prevents calculation outside 2022-2025', 1; + +-- Drop temporary table holding final result set +DROP TABLE IF EXISTS [#qcew_result_set]; + + +-- Calculate custom SANDAG employment categories using quarterly data -------- +SELECT + @year AS [year], + [fn_get_sandag_employment].[ownership_title], + [fn_get_sandag_employment].[industry_code], + 'jobs' AS [metric], + ROUND(SUM([month1_emplvl] + [month2_emplvl] + [month3_emplvl])/12.0, 0) AS [value] +--INTO [#qcew_result_set] +FROM [socioec_data].[bls].[qcew_by_area_quarterly] +INNER JOIN [socioec_data].[bls].[industry_code] + ON [qcew_by_area_quarterly].[naics_id] = [industry_code].[naics_id] +INNER JOIN [socioec_data].[bls].[ownership_titles] + ON [qcew_by_area_quarterly].[own_code] = [ownership_titles].[ownership_code] +CROSS APPLY [socioec_data].[bls].[fn_get_sandag_employment]([ownership_title], [industry_code]) +WHERE + [area_fips] = '06073' + AND [year] = @year + AND [fn_get_sandag_employment].[ownership_title] IS NOT NULL + AND [fn_get_sandag_employment].[industry_code] IS NOT NULL + -- Remove the "Private" ownership only categories + -- These are published directly in the annual QCEW + AND [fn_get_sandag_employment].[ownership_title] != 'Private' +GROUP BY + [fn_get_sandag_employment].[ownership_title], + [fn_get_sandag_employment].[industry_code] + + UNION ALL + +-- Calculate directly derived employment categories using annual data -------- +SELECT + @year AS [year], + [fn_get_sandag_employment].[ownership_title], + [fn_get_sandag_employment].[industry_code], + 'jobs' AS [metric], + ROUND(SUM([annual_avg_emplvl]), 0) AS [value] +FROM [socioec_data].[bls].[qcew_by_area_annual] +INNER JOIN [socioec_data].[bls].[industry_code] + ON [qcew_by_area_annual].[naics_id] = [industry_code].[naics_id] +INNER JOIN [socioec_data].[bls].[ownership_titles] + ON [qcew_by_area_annual].[own_code] = [ownership_titles].[ownership_code] +CROSS APPLY [socioec_data].[bls].[fn_get_sandag_employment]([ownership_title], [industry_code]) +WHERE + [area_fips] = '06073' + AND [year] = @year + AND [fn_get_sandag_employment].[ownership_title] IS NOT NULL + AND [fn_get_sandag_employment].[industry_code] IS NOT NULL + -- "Private" ownership only categories + -- Are published directly in the annual QCEW + AND [fn_get_sandag_employment].[ownership_title] = 'Private' +GROUP BY + [fn_get_sandag_employment].[ownership_title], + [fn_get_sandag_employment].[industry_code] + +ORDER BY + [ownership_title], + [industry_code] \ No newline at end of file diff --git a/sql/employment/get_region_self_emp.sql b/sql/employment/get_region_self_emp.sql deleted file mode 100644 index bf9e41a..0000000 --- a/sql/employment/get_region_self_emp.sql +++ /dev/null @@ -1,49 +0,0 @@ --- Get the ACS 1-year self-employed total for San Diego County - --- Initialize parameters ----------------------------------------------------- -DECLARE @year INTEGER = :year; -DECLARE @msg nvarchar(31) = 'ACS 1-Year Table does not exist'; - --- Send error message if no data exists -------------------------------------- -IF NOT EXISTS ( - SELECT TOP (1) * - FROM [acs].[detailed].[tables] - WHERE - [name] = 'B24080' - AND [year] = @year - AND - ( - [tables].[product] = '1Y' - OR ([tables].[year] = 2020 AND [tables].[product] = '5Y') - ) -) -SELECT @msg AS [msg] -ELSE -BEGIN - - -- Get the ACS data -------------------------------------------------------- - SELECT - @year AS [year], - 'SE' AS [industry_code], -- assign to 'SE' (Self Employed) NAICS category - 'jobs' AS [metric], - SUM([value]) AS [value] - FROM [acs].[detailed].[values] - INNER JOIN [acs].[detailed].[variables] - ON [values].[table_id] = [variables].[table_id] - AND [variables].[variable] = [values].[variable] - INNER JOIN [acs].[detailed].[tables] - ON [values].[table_id] = [tables].[table_id] - WHERE - [tables].[name] = 'B24080' - -- there is no 1-year data release for 2020 - AND - ( - [tables].[product] = '1Y' - OR ([tables].[year] = 2020 AND [tables].[product] = '5Y') - ) - AND REPLACE([variables].[label], ':', '') IN ( - 'Estimate!!Total!!Male!!Self-employed in own not incorporated business workers', - 'Estimate!!Total!!Female!!Self-employed in own not incorporated business workers' - ) - AND [year] = @year -END \ No newline at end of file diff --git a/sql/employment/xref_block_to_mgra.sql b/sql/employment/xref_block_to_mgra.sql index 52e9671..afe4942 100644 --- a/sql/employment/xref_block_to_mgra.sql +++ b/sql/employment/xref_block_to_mgra.sql @@ -1,8 +1,8 @@ /* This query provides a many-to-many cross reference mapping 2020 Census Blocks to Series 15 MGRAs There are two cross references for separate use cases - 1) Cross reference based on EDD point-level jobs data within industry codes - 2) Cross reference based on EDD point-level jobs data without considering industry codes + 1) Cross reference based on EDD point-level jobs data within SANDAG employment categories + 2) Cross reference based on EDD point-level jobs data without considering SANDAG employment categories 3) Cross reference based on simple land area intersection Notes: @@ -12,7 +12,8 @@ Notes: need to be allocated to MGRAs. 2) Data prior to year 2017 is not present in the EDD view and must be queried directly from the source database table. Note there is no 2016 - data available. + data available nor is there ownership data for 2014. In both instances, + this query returns "EDD point-level data does not exist". 3) This must be run on the GIS server. */ @@ -29,33 +30,54 @@ BEGIN THROW 50000, 'EDD xref only valid for Series 15 MGRAs',1; END --- Create shell table of 2020 Census Block x Industry Code -DROP TABLE IF EXISTS [#tt_block_industry]; -SELECT [GEOID20] AS [block], [industry_code] -INTO [#tt_block_industry] +-- Create shell table of 2020 Census Block x Ownership Title x Industry Code +DROP TABLE IF EXISTS [#tt_block_category]; +SELECT [GEOID20] AS [block], [ownership_title], [industry_code] +INTO [#tt_block_category] FROM [GeoDepot].[sde].[CENSUSBLOCKS] CROSS JOIN ( SELECT DISTINCT - CASE - WHEN LEFT([naics_code], 2) IN ('31','32','33') THEN '31-33' - WHEN LEFT([naics_code], 2) IN ('44','45') THEN '44-45' - WHEN LEFT([naics_code], 2) IN ('48','49') THEN '48-49' - -- there are records in 2011-2012 tagged as "Self Employed" with [naics_code] = '72' - -- there are records in all years with [naics_code] = '999999' as a NULL placeholder - WHEN [naics_code] = '72' OR LEFT([naics_code], 2) = '99' THEN NULL - WHEN LEFT([naics_code], 2) = '72' THEN LEFT([naics_code], 3) - -- NULL records are mapped to NULL despite falling into this ELSE condition - ELSE LEFT([naics_code], 2) - END AS [industry_code] - FROM [EMPCORE].[ca_edd].[vi_ca_edd_employment] -) AS [industry_code] -WHERE [industry_code] IS NOT NULL; + [fn_get_sandag_employment].[ownership_title], + [fn_get_sandag_employment].[industry_code] + FROM ( + SELECT + CASE + WHEN [ownership].[description] = 'Federal government' THEN 'Federal Government' + WHEN [ownership].[description] = 'State government' THEN 'State Government' + WHEN [ownership].[description] = 'Local government' THEN 'Local Government' + WHEN [ownership].[description] = 'Private sector' THEN 'Private' + ELSE NULL + END AS [ownership_title], + CASE + WHEN LEFT([naics_code], 2) IN ('31','32','33') THEN '31-33' + WHEN LEFT([naics_code], 2) IN ('44','45') THEN '44-45' + WHEN LEFT([naics_code], 2) IN ('48','49') THEN '48-49' + -- there are records in 2011-2012 tagged as "Self Employed" with [naics_code] = '72' + -- there are records in all years with [naics_code] = '999999' as a NULL placeholder + WHEN [naics_code] = '72' OR LEFT([naics_code], 2) = '99' THEN NULL + WHEN LEFT([naics_code], 2) = '72' THEN LEFT([naics_code], 3) + -- NULL records are mapped to NULL despite falling into this ELSE condition + ELSE LEFT([naics_code], 2) + END AS [industry_code] + FROM [EMPCORE].[ca_edd].[vi_ca_edd_employment] + INNER JOIN [EMPCORE].[ca_edd].[ownership] + ON [vi_ca_edd_employment].[ownership_id] = [ownership].[ownership_id] + -- Filter year 2024 provides all 23 distinct employment categories + WHERE [year] = 2024 + ) AS [tt] + CROSS APPLY [EMPCORE].[ca_edd].[fn_get_sandag_employment]([tt].[ownership_title], [tt].[industry_code]) + WHERE + [fn_get_sandag_employment].[ownership_title] IS NOT NULL + AND [fn_get_sandag_employment].[industry_code] IS NOT NULL +) AS [sandag_employment_categories]; + -- Create temporary table for EDD data to support spatial index DROP TABLE IF EXISTS [#edd]; CREATE TABLE [#edd] ( [id] INTEGER IDENTITY(1,1) NOT NULL, - [industry_code] NVARCHAR(5) NULL, + [ownership_title] NVARCHAR(50) NOT NULL, + [industry_code] NVARCHAR(5) NOT NULL, [jobs] FLOAT NOT NULL, [Shape] GEOMETRY NOT NULL, CONSTRAINT [pk_tt_edd] PRIMARY KEY ([id]) @@ -80,11 +102,19 @@ IF @year >= 2017 BEGIN INSERT INTO [#edd] SELECT - [industry_code], + [fn_get_sandag_employment].[ownership_title], + [fn_get_sandag_employment].[industry_code], 1.0 * [emp_total]/[emp_valid] AS [jobs], [SHAPE] FROM ( SELECT + CASE + WHEN [ownership].[description] = 'Federal government' THEN 'Federal Government' + WHEN [ownership].[description] = 'State government' THEN 'State Government' + WHEN [ownership].[description] = 'Local government' THEN 'Local Government' + WHEN [ownership].[description] = 'Private sector' THEN 'Private' + ELSE NULL + END AS [ownership_title], CASE WHEN LEFT([naics_code], 2) IN ('31','32','33') THEN '31-33' WHEN LEFT([naics_code], 2) IN ('44','45') THEN '44-45' @@ -125,93 +155,140 @@ BEGIN AS [emp_total], [SHAPE] FROM [EMPCORE].[ca_edd].[vi_ca_edd_employment] + INNER JOIN [EMPCORE].[ca_edd].[ownership] + ON [vi_ca_edd_employment].[ownership_id] = [ownership].[ownership_id] WHERE [year] = @year ) AS [tt] + CROSS APPLY [EMPCORE].[ca_edd].[fn_get_sandag_employment]([tt].[ownership_title], [tt].[industry_code]) WHERE [emp_valid] > 0 AND [emp_total] > 0 + AND [fn_get_sandag_employment].[ownership_title] IS NOT NULL + AND [fn_get_sandag_employment].[industry_code] IS NOT NULL END ELSE IF @year BETWEEN 2010 AND 2013 BEGIN INSERT INTO [#edd] SELECT - CASE - WHEN LEFT([code], 2) IN ('31','32','33') THEN '31-33' - WHEN LEFT([code], 2) IN ('44','45') THEN '44-45' - WHEN LEFT([code], 2) IN ('48','49') THEN '48-49' - -- there are records in 2011-2012 tagged as "Self Employed" with [code] = '72' - -- there are records in all years with [code] = '999999' as a NULL placeholder - WHEN [code] = '72' OR LEFT([code], 2) = '99' THEN NULL - WHEN LEFT([code], 2) = '72' THEN LEFT([code], 3) - -- NULL records are mapped to NULL despite falling into this ELSE condition - -- Keep these records for total EDD jobs xref even if industry code is NULL - ELSE LEFT([code], 2) - END AS [industry_code], - [employment] * ISNULL([headquarters].[share], 1) AS [jobs], - ISNULL([headquarters].[shape], [businesses].[shape]) AS [SHAPE] - FROM [EMPCORE].[ca_edd].[businesses] - LEFT OUTER JOIN [EMPCORE].[ca_edd].[naics] - ON [businesses].[naics_id] = [naics].[naics_id] - LEFT JOIN [EMPCORE].[ca_edd].[headquarters] - ON [businesses].[year] = [headquarters].[year] - AND [businesses].[emp_id] = [headquarters].[emp_id] - INNER JOIN ( - SELECT [year], [emp_id], [employment] - FROM [EMPCORE].[ca_edd].[employment] - WHERE - [month_id] = 14 -- adjusted employment - AND [employment] > 0 - AND [year] = @year - ) AS [employment] - ON [businesses].[year] = [employment].[year] - AND [businesses].[emp_id] = [employment].[emp_id] + [fn_get_sandag_employment].[ownership_title], + [fn_get_sandag_employment].[industry_code], + [jobs], + [SHAPE] + FROM ( + SELECT + CASE + WHEN [ownership].[description] = 'Federal government' THEN 'Federal Government' + WHEN [ownership].[description] = 'State government' THEN 'State Government' + WHEN [ownership].[description] = 'Local government' THEN 'Local Government' + WHEN [ownership].[description] = 'Private sector' THEN 'Private' + ELSE NULL + END AS [ownership_title], + CASE + WHEN LEFT([code], 2) IN ('31','32','33') THEN '31-33' + WHEN LEFT([code], 2) IN ('44','45') THEN '44-45' + WHEN LEFT([code], 2) IN ('48','49') THEN '48-49' + -- there are records in 2011-2012 tagged as "Self Employed" with [code] = '72' + -- there are records in all years with [code] = '999999' as a NULL placeholder + WHEN [code] = '72' OR LEFT([code], 2) = '99' THEN NULL + WHEN LEFT([code], 2) = '72' THEN LEFT([code], 3) + -- NULL records are mapped to NULL despite falling into this ELSE condition + -- Keep these records for total EDD jobs xref even if industry code is NULL + ELSE LEFT([code], 2) + END AS [industry_code], + [employment] * ISNULL([headquarters].[share], 1) AS [jobs], + ISNULL([headquarters].[shape], [businesses].[shape]) AS [SHAPE] + FROM [EMPCORE].[ca_edd].[businesses] + LEFT OUTER JOIN [EMPCORE].[ca_edd].[naics] + ON [businesses].[naics_id] = [naics].[naics_id] + LEFT JOIN [EMPCORE].[ca_edd].[headquarters] + ON [businesses].[year] = [headquarters].[year] + AND [businesses].[emp_id] = [headquarters].[emp_id] + INNER JOIN ( + SELECT [year], [emp_id], [employment] + FROM [EMPCORE].[ca_edd].[employment] + WHERE + [month_id] = 14 -- adjusted employment + AND [employment] > 0 + AND [year] = @year + ) AS [employment] + ON [businesses].[year] = [employment].[year] + AND [businesses].[emp_id] = [employment].[emp_id] + INNER JOIN [EMPCORE].[ca_edd].[ownership] + ON [businesses].[ownership_id] = [ownership].[ownership_id] + ) AS [tt] + CROSS APPLY [EMPCORE].[ca_edd].[fn_get_sandag_employment]([tt].[ownership_title], [tt].[industry_code]) + WHERE + [jobs] > 0 + AND [fn_get_sandag_employment].[ownership_title] IS NOT NULL + AND [fn_get_sandag_employment].[industry_code] IS NOT NULL END ELSE IF @year BETWEEN 2014 AND 2016 BEGIN INSERT INTO [#edd] SELECT - CASE - WHEN LEFT([code], 2) IN ('31','32','33') THEN '31-33' - WHEN LEFT([code], 2) IN ('44','45') THEN '44-45' - WHEN LEFT([code], 2) IN ('48','49') THEN '48-49' - -- there are records in 2011-2012 tagged as "Self Employed" with [code] = '72' - -- there are records in all years with [code] = '999999' as a NULL placeholder - WHEN [code] = '72' OR LEFT([code], 2) = '99' THEN NULL - WHEN LEFT([code], 2) = '72' THEN LEFT([code], 3) - -- NULL records are mapped to NULL despite falling into this ELSE condition - -- Keep these records for total EDD jobs xref even if industry code is NULL - ELSE LEFT([code], 2) - END AS [industry_code], - [employment] * ISNULL([headquarters].[share], 1) AS [jobs], - ISNULL([headquarters].[shape], [businesses].[shape]) AS [SHAPE] - FROM [EMPCORE].[ca_edd].[businesses] - LEFT OUTER JOIN [EMPCORE].[ca_edd].[naics] - ON [businesses].[naics_id] = [naics].[naics_id] - LEFT JOIN [EMPCORE].[ca_edd].[headquarters] - ON [businesses].[year] = [headquarters].[year] - AND [businesses].[emp_id] = [headquarters].[emp_id] - INNER JOIN ( + [fn_get_sandag_employment].[ownership_title], + [fn_get_sandag_employment].[industry_code], + [jobs], + [SHAPE] + FROM ( SELECT - [year], - [emp_id], - -- 15, 16, 17 are mpnths from [employment] table where data was - -- stored in [emp1], [emp2], [emp3] but actual month unknown - -- check [EMPCORE].[ca_edd].[month] for more detail - 1.0 * ((ISNULL([15], 0) + ISNULL([16], 0) + ISNULL([17], 0)) - / - (CASE WHEN [15] IS NOT NULL THEN 1 ELSE 0 END - + CASE WHEN [16] IS NOT NULL THEN 1 ELSE 0 END - + CASE WHEN [17] IS NOT NULL THEN 1 ELSE 0 END - )) - AS [employment] - FROM [EMPCORE].[ca_edd].[employment] - PIVOT(SUM([employment]) FOR [month_id] IN ([15], [16], [17])) AS [pivot] - WHERE - [year] = @year AND - ([15] IS NOT NULL OR [16] IS NOT NULL OR [17] IS NOT NULL) - ) AS [employment] - ON [businesses].[year] = [employment].[year] - AND [businesses].[emp_id] = [employment].[emp_id] + CASE + WHEN [ownership].[description] = 'Federal government' THEN 'Federal Government' + WHEN [ownership].[description] = 'State government' THEN 'State Government' + WHEN [ownership].[description] = 'Local government' THEN 'Local Government' + WHEN [ownership].[description] = 'Private sector' THEN 'Private' + ELSE NULL + END AS [ownership_title], + CASE + WHEN LEFT([code], 2) IN ('31','32','33') THEN '31-33' + WHEN LEFT([code], 2) IN ('44','45') THEN '44-45' + WHEN LEFT([code], 2) IN ('48','49') THEN '48-49' + -- there are records in 2011-2012 tagged as "Self Employed" with [code] = '72' + -- there are records in all years with [code] = '999999' as a NULL placeholder + WHEN [code] = '72' OR LEFT([code], 2) = '99' THEN NULL + WHEN LEFT([code], 2) = '72' THEN LEFT([code], 3) + -- NULL records are mapped to NULL despite falling into this ELSE condition + -- Keep these records for total EDD jobs xref even if industry code is NULL + ELSE LEFT([code], 2) + END AS [industry_code], + [employment] * ISNULL([headquarters].[share], 1) AS [jobs], + ISNULL([headquarters].[shape], [businesses].[shape]) AS [SHAPE] + FROM [EMPCORE].[ca_edd].[businesses] + LEFT OUTER JOIN [EMPCORE].[ca_edd].[naics] + ON [businesses].[naics_id] = [naics].[naics_id] + LEFT JOIN [EMPCORE].[ca_edd].[headquarters] + ON [businesses].[year] = [headquarters].[year] + AND [businesses].[emp_id] = [headquarters].[emp_id] + INNER JOIN ( + SELECT + [year], + [emp_id], + -- 15, 16, 17 are mpnths from [employment] table where data was + -- stored in [emp1], [emp2], [emp3] but actual month unknown + -- check [EMPCORE].[ca_edd].[month] for more detail + 1.0 * ((ISNULL([15], 0) + ISNULL([16], 0) + ISNULL([17], 0)) + / + (CASE WHEN [15] IS NOT NULL THEN 1 ELSE 0 END + + CASE WHEN [16] IS NOT NULL THEN 1 ELSE 0 END + + CASE WHEN [17] IS NOT NULL THEN 1 ELSE 0 END + )) + AS [employment] + FROM [EMPCORE].[ca_edd].[employment] + PIVOT(SUM([employment]) FOR [month_id] IN ([15], [16], [17])) AS [pivot] + WHERE + [year] = @year AND + ([15] IS NOT NULL OR [16] IS NOT NULL OR [17] IS NOT NULL) + ) AS [employment] + ON [businesses].[year] = [employment].[year] + AND [businesses].[emp_id] = [employment].[emp_id] + INNER JOIN [EMPCORE].[ca_edd].[ownership] + ON [businesses].[ownership_id] = [ownership].[ownership_id] + ) AS [tt] + CROSS APPLY [EMPCORE].[ca_edd].[fn_get_sandag_employment]([tt].[ownership_title], [tt].[industry_code]) + WHERE + [jobs] > 0 + AND [fn_get_sandag_employment].[ownership_title] IS NOT NULL + AND [fn_get_sandag_employment].[industry_code] IS NOT NULL END @@ -223,22 +300,23 @@ SELECT @msg AS [msg] ELSE -- Build cross reference of Census 2020 Blocks to MGRAs ---------------------- BEGIN - -- Calculate % allocation of Census 2020 Block jobs to MGRAs within Industry Code - WITH [xref_industry] AS ( + -- Calculate % allocation of Census 2020 Block jobs to MGRAs within SANDAG employment category + WITH [xref_edd_category] AS ( SELECT [CENSUSBLOCKS].[GEOID20] AS [block], + [ownership_title], [industry_code], [MGRA15].[MGRA] AS [mgra], SUM([jobs]) - / SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20], [industry_code]) - AS [pct_industry] + / SUM(SUM([jobs])) OVER (PARTITION BY [CENSUSBLOCKS].[GEOID20], [ownership_title], [industry_code]) + AS [pct_edd_category] FROM [#edd] INNER JOIN [GeoDepot].[sde].[CENSUSBLOCKS] ON [#edd].[Shape].STIntersects([CENSUSBLOCKS].[Shape]) = 1 INNER JOIN [GeoDepot].[sde].[MGRA15] ON [#edd].[Shape].STIntersects([MGRA15].[Shape]) = 1 GROUP BY - [CENSUSBLOCKS].[GEOID20], [industry_code], [MGRA15].[MGRA] + [CENSUSBLOCKS].[GEOID20], [ownership_title], [industry_code], [MGRA15].[MGRA] ), -- Calculate % allocation of Census 2020 Block jobs to MGRAs [xref_edd] AS ( @@ -277,33 +355,36 @@ BEGIN / [CENSUSBLOCKS].[Shape].STArea()) > 0.01 ) AS [raw_xref_area] ) - -- Combine results and set flag indicating which xref to use within Block x Industry Code + -- Combine results and set flag indicating which xref to use within block x SANDAG employment category SELECT - [#tt_block_industry].[block], + [#tt_block_category].[block], [xref_area].[mgra], - [#tt_block_industry].[industry_code], - [pct_industry], + [#tt_block_category].[ownership_title], + [#tt_block_category].[industry_code], + [pct_edd_category], [pct_edd], [pct_area], CASE - WHEN COUNT([pct_industry]) OVER (PARTITION BY [#tt_block_industry].[block], [#tt_block_industry].[industry_code]) > 0 THEN 'pct_industry' - WHEN COUNT([pct_edd]) OVER (PARTITION BY [#tt_block_industry].[block], [#tt_block_industry].[industry_code]) > 0 THEN 'pct_edd' + WHEN COUNT([pct_edd_category]) OVER (PARTITION BY [#tt_block_category].[block], [#tt_block_category].[ownership_title], [#tt_block_category].[industry_code]) > 0 THEN 'pct_edd_category' + WHEN COUNT([pct_edd]) OVER (PARTITION BY [#tt_block_category].[block]) > 0 THEN 'pct_edd' ELSE 'pct_area' END AS [flag] - FROM [#tt_block_industry] + FROM [#tt_block_category] LEFT OUTER JOIN [xref_area] - ON [#tt_block_industry].[block] = [xref_area].[block] + ON [#tt_block_category].[block] = [xref_area].[block] LEFT OUTER JOIN [xref_edd] - ON [#tt_block_industry].[block] = [xref_edd].[block] + ON [#tt_block_category].[block] = [xref_edd].[block] AND [xref_area].[mgra] = [xref_edd].[mgra] - LEFT OUTER JOIN [xref_industry] - ON [#tt_block_industry].[block] = [xref_industry].[block] - AND [#tt_block_industry].[industry_code] = [xref_industry].[industry_code] - AND [xref_area].[mgra] = [xref_industry].[mgra] + LEFT OUTER JOIN [xref_edd_category] + ON [#tt_block_category].[block] = [xref_edd_category].[block] + AND [#tt_block_category].[ownership_title] = [xref_edd_category].[ownership_title] + AND [#tt_block_category].[industry_code] = [xref_edd_category].[industry_code] + AND [xref_area].[mgra] = [xref_edd_category].[mgra] ORDER BY - [#tt_block_industry].[block], + [#tt_block_category].[block], [xref_area].[mgra], - [#tt_block_industry].[industry_code] + [#tt_block_category].[industry_code] END +DROP TABLE IF EXISTS [#tt_block_category]; DROP TABLE IF EXISTS [#edd]; \ No newline at end of file diff --git a/sql/employment/xref_se_to_mgra.sql b/sql/employment/xref_se_to_mgra.sql deleted file mode 100644 index c0f9482..0000000 --- a/sql/employment/xref_se_to_mgra.sql +++ /dev/null @@ -1,118 +0,0 @@ -/* -This query creates a cross reference from census blockgroup (2013+) or tract -(2010-2012) to SANDAG MGRA to allocate self employment counts using the -following methodology. - - 1). The percentage of 18-64 year olds across MGRAs within each geography - after removing 'Group Quarters - Institutional Correctional Facilities' - and 'Group Quarters - Military' persons. - 2). The percentage of all persons across MGRAs within each geography - 3). An equal split across MGRAs within each geography -*/ - --- Initialize parameters ----------------------------------------------------- -DECLARE @run_id INTEGER = :run_id; -DECLARE @year INTEGER = :year; -DECLARE @series INTEGER = (SELECT [series] FROM [metadata].[run] WHERE [run_id] = @run_id); - --- Send error message if no data exists -------------------------------------- -IF NOT EXISTS ( - SELECT TOP (1) * - FROM [outputs].[ase] - WHERE - [run_id] = @run_id - AND [year] = @year -) -BEGIN - THROW 50000, 'Age/Sex/Ethnicity data does not exist for this run.', 1; -END -ELSE -BEGIN - - -- Get MGRA cross reference ---------------------------------------------- - -- Population aged 18-64 excluding Military and Prisons - WITH [18_64] AS ( - SELECT - [mgra], - SUM([value]) AS [value] - FROM [outputs].[ase] - WHERE - [run_id] = @run_id - AND [year] = @year - AND [value] > 0 - AND [pop_type] NOT IN ( - 'Group Quarters - Military', - 'Group Quarters - Institutional Correctional Facilities' - ) - AND [age_group] IN ( - '18 and 19', - '20 to 24', - '25 to 29', - '30 to 34', - '35 to 39', - '40 to 44', - '45 to 49', - '50 to 54', - '55 to 59', - '60 and 61', - '62 to 64' - ) - GROUP BY [mgra] - ), - -- Total population without exclusions - [pop] AS ( - SELECT - [mgra], - SUM([value]) AS [value] - FROM [outputs].[ase] - WHERE - [run_id] = @run_id - AND [year] = @year - AND [value] > 0 - GROUP BY [mgra] - ), - -- Exhaustive list of MGRAs and census blockgroups or tracts - [mgras] AS ( - SELECT - [mgra], - CASE - WHEN @year BETWEEN 2010 AND 2012 THEN [tract] - ELSE [blockgroup] - END AS [geography] - FROM [demographic_warehouse].[dim].[mgra] - INNER JOIN [demographic_warehouse].[dim].[mgra_xref] - ON [mgra].[mgra_id] = [mgra_xref].[mgra_id] - AND [mgra_xref].[xref_year] = @year - WHERE [mgra].[series] = @series - ) - -- Return cross reference with flag field indicating which to use - SELECT - [geography], - [mgras].[mgra], - COALESCE( - 1.0 * [18_64].[value] - / SUM([18_64].[value]) OVER (PARTITION BY [geography]) - , 0) - AS [pct_18_64], - COALESCE( - 1.0 * [pop].[value] - / SUM([pop].[value]) OVER (PARTITION BY [geography]) - , 0) - AS [pct_pop], - 1.0 / COUNT([geography]) OVER (PARTITION BY [geography]) AS [pct_split], - CASE - WHEN SUM([18_64].[value]) OVER (PARTITION BY [geography]) > 0 - THEN 'pct_18_64' - WHEN SUM([pop].[value]) OVER (PARTITION BY [geography]) > 0 - THEN 'pct_pop' - ELSE 'pct_split' - END AS [flag] - FROM [mgras] - LEFT OUTER JOIN [18_64] - ON [mgras].[mgra] = [18_64].[mgra] - LEFT OUTER JOIN [pop] - ON [mgras].[mgra] = [pop].[mgra] - ORDER BY - [geography], - [mgras].[mgra] -END \ No newline at end of file