Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Onboard USA names dataset #96

Merged
merged 9 commits into from
Jul 21, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
28 changes: 28 additions & 0 deletions datasets/usa_names/_terraform/provider.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/


provider "google" {
project = var.project_id
impersonate_service_account = var.impersonating_acct
region = var.region
}

data "google_client_openid_userinfo" "me" {}

output "impersonating-account" {
value = data.google_client_openid_userinfo.me.email
}
39 changes: 39 additions & 0 deletions datasets/usa_names/_terraform/usa_1910_current_pipeline.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/


resource "google_bigquery_table" "usa_1910_current" {
project = var.project_id
dataset_id = "usa_names"
table_id = "usa_1910_current"

description = "The table contains the number of applicants for a Social Security card by year of birth and sex. The number of such applicants is restricted to U.S. births where the year of birth, sex, State of birth (50 States and District of Columbia) are known, and where the given name is at least 2 characters long.\n\nsource: http://www.ssa.gov/OACT/babynames/limits.html"




depends_on = [
google_bigquery_dataset.usa_names
]
}

output "bigquery_table-usa_1910_current-table_id" {
value = google_bigquery_table.usa_1910_current.table_id
}

output "bigquery_table-usa_1910_current-id" {
value = google_bigquery_table.usa_1910_current.id
}
26 changes: 26 additions & 0 deletions datasets/usa_names/_terraform/usa_names_dataset.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/


resource "google_bigquery_dataset" "usa_names" {
dataset_id = "usa_names"
project = var.project_id
description = "This public dataset was created by the Social Security Administration and contains all names from Social Security card applications for births that occurred in the United States after 1879. Note that many people born before 1937 never applied for a Social Security card, so their names are not included in this data. For others who did apply, records may not show the place of birth, and again their names are not included in the data.\n\nAll data are from a 100% sample of records on Social Security card applications as of the end of February 2015. To safeguard privacy, the Social Security Administration restricts names to those with at least 5 occurrences."
}

output "bigquery_dataset-usa_names-dataset_id" {
value = google_bigquery_dataset.usa_names.dataset_id
}
23 changes: 23 additions & 0 deletions datasets/usa_names/_terraform/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
/**
* Copyright 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/


variable "project_id" {}
variable "bucket_name_prefix" {}
variable "impersonating_acct" {}
variable "region" {}
variable "env" {}

30 changes: 30 additions & 0 deletions datasets/usa_names/dataset.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

dataset:
name: usa_names

friendly_name: ~
description: ~
update_frequency: ~
dataset_sources: ~
terms_of_use: ~

resources:
- type: bigquery_dataset
dataset_id: usa_names
description: |-
This public dataset was created by the Social Security Administration and contains all names from Social Security card applications for births that occurred in the United States after 1879. Note that many people born before 1937 never applied for a Social Security card, so their names are not included in this data. For others who did apply, records may not show the place of birth, and again their names are not included in the data.

All data are from a 100% sample of records on Social Security card applications as of the end of February 2015. To safeguard privacy, the Social Security Administration restricts names to those with at least 5 occurrences.
83 changes: 83 additions & 0 deletions datasets/usa_names/usa_1910_current/pipeline.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

---
resources:
- type: bigquery_table
table_id: usa_1910_current
description: |-
The table contains the number of applicants for a Social Security card by year of birth and sex. The number of such applicants is restricted to U.S. births where the year of birth, sex, State of birth (50 States and District of Columbia) are known, and where the given name is at least 2 characters long.

source: http://www.ssa.gov/OACT/babynames/limits.html

dag:
initialize:
dag_id: "usa_1910_current"
default_args:
owner: "Google"
depends_on_past: False
start_date: '2021-06-17'
max_active_runs: 1
schedule_interval: "@monthly"
catchup: False
default_view: graph

tasks:
- operator: "BashOperator"
description: "Task to copy `namesbystate.zip` from Social Security Administration to GCS"
args:
task_id: "download_and_process_source_zip_file"
bash_command: |
mkdir -p $data_dir/{{ ds }}
curl -o $data_dir/{{ ds }}/namesbystate.zip -L $zip_source_url
unzip $data_dir/{{ ds }}/namesbystate.zip -d $data_dir/{{ ds }}
cat $data_dir/{{ ds }}/*.TXT >> $data_dir/{{ ds }}/data.csv
env:
zip_source_url: "https://www.ssa.gov/OACT/babynames/state/namesbystate.zip"
data_dir: "/home/airflow/gcs/data/usa_names/usa_1910_current"

- operator: "GoogleCloudStorageToBigQueryOperator"
description: "Task to load the data from Airflow data folder to BigQuery"
args:
task_id: "load_csv_file_to_bq_table"
bucket: "{{ var.json.shared.composer_bucket }}"
source_objects: ["data/usa_names/usa_1910_current/{{ ds }}/data.csv"]
source_format: "CSV"
destination_project_dataset_table: "usa_names.usa_1910_current"
skip_leading_rows: 0
write_disposition: "WRITE_TRUNCATE"
schema_fields:
- name: state
type: STRING
description: 2-digit state code
mode: NULLABLE
- name: gender
type: STRING
description: Sex (M=male or F=female)
mode: NULLABLE
- name: year
type: INTEGER
description: 4-digit year of birth
mode: NULLABLE
- name: name
type: STRING
description: Given name of a person at birth
mode: NULLABLE
- name: number
type: INTEGER
description: Number of occurrences of the name
mode: NULLABLE

graph_paths:
- "download_and_process_source_zip_file >> load_csv_file_to_bq_table"
89 changes: 89 additions & 0 deletions datasets/usa_names/usa_1910_current/usa_1910_current_dag.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


from airflow import DAG
from airflow.contrib.operators import gcs_to_bq
from airflow.operators import bash_operator

default_args = {
"owner": "Google",
"depends_on_past": False,
"start_date": "2021-06-17",
}


with DAG(
dag_id="usa_names.usa_1910_current",
default_args=default_args,
max_active_runs=1,
schedule_interval="@monthly",
catchup=False,
default_view="graph",
) as dag:

# Task to copy `namesbystate.zip` from Social Security Administration to GCS
download_and_process_source_zip_file = bash_operator.BashOperator(
task_id="download_and_process_source_zip_file",
bash_command="mkdir -p $data_dir/{{ ds }}\ncurl -o $data_dir/{{ ds }}/namesbystate.zip -L $zip_source_url\nunzip $data_dir/{{ ds }}/namesbystate.zip -d $data_dir/{{ ds }}\ncat $data_dir/{{ ds }}/*.TXT \u003e\u003e $data_dir/{{ ds }}/data.csv\n",
env={
"zip_source_url": "https://www.ssa.gov/OACT/babynames/state/namesbystate.zip",
"data_dir": "/home/airflow/gcs/data/usa_names/usa_1910_current",
},
)

# Task to load the data from Airflow data folder to BigQuery
load_csv_file_to_bq_table = gcs_to_bq.GoogleCloudStorageToBigQueryOperator(
task_id="load_csv_file_to_bq_table",
bucket="{{ var.json.shared.composer_bucket }}",
source_objects=["data/usa_names/usa_1910_current/{{ ds }}/data.csv"],
source_format="CSV",
destination_project_dataset_table="usa_names.usa_1910_current",
skip_leading_rows=0,
write_disposition="WRITE_TRUNCATE",
schema_fields=[
{
"name": "state",
"type": "STRING",
"description": "2-digit state code",
"mode": "NULLABLE",
},
{
"name": "gender",
"type": "STRING",
"description": "Sex (M=male or F=female)",
"mode": "NULLABLE",
},
{
"name": "year",
"type": "INTEGER",
"description": "4-digit year of birth",
"mode": "NULLABLE",
},
{
"name": "name",
"type": "STRING",
"description": "Given name of a person at birth",
"mode": "NULLABLE",
},
{
"name": "number",
"type": "INTEGER",
"description": "Number of occurrences of the name",
"mode": "NULLABLE",
},
],
)

download_and_process_source_zip_file >> load_csv_file_to_bq_table