Skip to content

Commit

Permalink
feat(dbt-cloud): allow jobs to be retrieved from project id (#11673)
Browse files Browse the repository at this point in the history
### Summary & Motivation
As the title.

### How I Tested These Changes
pytest, local
  • Loading branch information
rexledesma committed Jan 13, 2023
1 parent 66c4e7d commit 429c414
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 35 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,30 @@ def make_request(

raise Failure(f"Max retries ({self._request_max_retries}) exceeded with url: {url}.")

def list_jobs(
self, project_id: int, order_by: Optional[str] = "-id"
) -> Sequence[Mapping[str, Any]]:
"""
List all dbt jobs in a dbt Cloud project.
Args:
project_id (int): The ID of the relevant dbt Cloud project. You can find this value by
going to your account settings in the dbt Cloud UI. It will be the final
number in the url, e.g.: ``https://cloud.getdbt.com/next/settings/accounts/{account_id}/projects/{project_id}/``
order_by (Optional[str]): An identifier designated by dbt Cloud in which to sort the
results before returning them. Useful when combined with offset and limit to load
runs for a job. Defaults to "-id" where "-" designates reverse order and "id" is
the key to filter on.
Returns:
List[Dict[str, Any]]: Parsed json data from the response to this request
"""
return self.make_request(
"GET",
f"{self._account_id}/jobs",
params={"project_id": project_id, "order_by": order_by},
)

def get_job(self, job_id: int) -> Mapping[str, Any]:
"""
Gets details about a given dbt job from the dbt Cloud API.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
SAMPLE_ACCOUNT_ID,
SAMPLE_API_PREFIX,
SAMPLE_JOB_ID,
SAMPLE_PROJECT_ID,
SAMPLE_RUN_ID,
sample_job_details,
sample_list_artifacts,
sample_list_job_details,
sample_run_details,
sample_run_results,
sample_runs_details,
Expand All @@ -27,6 +29,14 @@ def get_dbt_cloud_resource(**kwargs):
)


@responses.activate
def test_list_jobs():
dc_resource = get_dbt_cloud_resource()

responses.add(responses.GET, f"{SAMPLE_API_PREFIX}/jobs", json=sample_list_job_details())
assert dc_resource.list_jobs(SAMPLE_PROJECT_ID) == sample_list_job_details()["data"]


def test_get_job():
dc_resource = get_dbt_cloud_resource()

Expand Down
Original file line number Diff line number Diff line change
@@ -1,55 +1,76 @@
from dagster._utils.merger import deep_merge_dicts

SAMPLE_ACCOUNT_ID = 30000
SAMPLE_PROJECT_ID = 35000
SAMPLE_JOB_ID = 40000
SAMPLE_JOB_ID_TWO = 40001
SAMPLE_RUN_ID = 5000000

SAMPLE_API_PREFIX = f"https://cloud.getdbt.com/api/v2/accounts/{SAMPLE_ACCOUNT_ID}"


def sample_job_details():
def job_details_data(job_id: int):
return {
"execution": {"timeout_seconds": 0},
"generate_docs": False,
"run_generate_sources": False,
"id": job_id,
"account_id": SAMPLE_ACCOUNT_ID,
"project_id": 50000,
"environment_id": 47000,
"name": "MyCoolJob",
"dbt_version": None,
"created_at": "2021-10-29T21:35:33.278228+00:00",
"updated_at": "2021-11-01T22:47:48.056913+00:00",
"execute_steps": ["dbt run"],
"state": 1,
"deferring_job_definition_id": None,
"lifecycle_webhooks": False,
"lifecycle_webhooks_url": None,
"triggers": {
"github_webhook": False,
"git_provider_webhook": False,
"custom_branch_only": False,
"schedule": False,
},
"settings": {"threads": 4, "target_name": "default"},
"schedule": {
"cron": "0 * * * *",
"date": {"type": "every_day"},
"time": {"type": "every_hour", "interval": 1},
},
"is_deferrable": False,
"generate_sources": False,
"cron_humanized": "Every hour",
"next_run": None,
"next_run_humanized": None,
}


def sample_list_job_details():
return {
"status": {
"code": 200,
"is_success": True,
"user_message": "Success!",
"developer_message": "",
},
"data": {
"execution": {"timeout_seconds": 0},
"generate_docs": False,
"run_generate_sources": False,
"id": SAMPLE_JOB_ID,
"account_id": SAMPLE_ACCOUNT_ID,
"project_id": 50000,
"environment_id": 47000,
"name": "MyCoolJob",
"dbt_version": None,
"created_at": "2021-10-29T21:35:33.278228+00:00",
"updated_at": "2021-11-01T22:47:48.056913+00:00",
"execute_steps": ["dbt run"],
"state": 1,
"deferring_job_definition_id": None,
"lifecycle_webhooks": False,
"lifecycle_webhooks_url": None,
"triggers": {
"github_webhook": False,
"git_provider_webhook": False,
"custom_branch_only": False,
"schedule": False,
},
"settings": {"threads": 4, "target_name": "default"},
"schedule": {
"cron": "0 * * * *",
"date": {"type": "every_day"},
"time": {"type": "every_hour", "interval": 1},
},
"is_deferrable": False,
"generate_sources": False,
"cron_humanized": "Every hour",
"next_run": None,
"next_run_humanized": None,
"data": [
job_details_data(job_id=SAMPLE_JOB_ID),
job_details_data(job_id=SAMPLE_JOB_ID_TWO),
],
}


def sample_job_details():
return {
"status": {
"code": 200,
"is_success": True,
"user_message": "Success!",
"developer_message": "",
},
"data": [job_details_data(job_id=SAMPLE_JOB_ID)],
}


Expand Down

0 comments on commit 429c414

Please sign in to comment.