Skip to content

Commit

Permalink
Change param http_params to dict type in task http (#130)
Browse files Browse the repository at this point in the history
  • Loading branch information
HarshitNagpal29 committed Jan 9, 2024
1 parent f16599f commit 0f120f6
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 3 deletions.
2 changes: 2 additions & 0 deletions UPDATING.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ It started after version 2.0.5 released
* Remove attribute tenant from pydolphinscheduler.core.workflow.workflow ([#54](https://github.com/apache/dolphinscheduler-sdk-python/pull/54))
and please change tenant name in ``config.yaml`` in ``PYDS_HOME``
* Drop support of python3.6 and python3.7 ([#126](https://github.com/apache/dolphinscheduler-sdk-python/pull/126))
* Change parameter `http_params` to dict type for easy to use in task http.([#130](https://github.com/apache/dolphinscheduler-sdk-python/pull/130))


## 4.0.0

Expand Down
49 changes: 46 additions & 3 deletions src/pydolphinscheduler/tasks/http.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@

from __future__ import annotations

import warnings

from pydolphinscheduler.constants import TaskType
from pydolphinscheduler.core.parameter import Direction, ParameterHelper
from pydolphinscheduler.core.task import Task
from pydolphinscheduler.exceptions import PyDSParamException

Expand Down Expand Up @@ -51,7 +54,30 @@ class HttpCheckCondition:


class Http(Task):
"""Task HTTP object, declare behavior for HTTP task to dolphinscheduler."""
"""Task HTTP object, declare behavior for HTTP task to dolphinscheduler.
:param name: The name or identifier for the HTTP task.
:param url: The URL endpoint for the HTTP request.
:param http_method: The HTTP method for the request (GET, POST, etc.). Defaults to HttpMethod.GET.
:param http_params: Parameters for the HTTP request. Defaults to None.
:param http_check_condition: Condition for checking the HTTP response status.
Defaults to HttpCheckCondition.STATUS_CODE_DEFAULT.
:param condition: Additional condition to evaluate if `http_check_condition` is not STATUS_CODE_DEFAULT.
:param connect_timeout: Connection timeout for the HTTP request in milliseconds. Defaults to 60000.
:param socket_timeout: Socket timeout for the HTTP request in milliseconds. Defaults to 60000.
Attributes:
_task_custom_attr (set): A set containing custom attributes specific to the Http task,
including 'url', 'http_method', 'http_params', and more.
Raises:
PyDSParamException: Exception raised for invalid parameters, such as unsupported HTTP methods or conditions.
Example:
Usage example for creating an HTTP task:
http_task = Http(name="http_task", url="https://api.example.com", http_method="POST", http_params={"key": "value"})
"""

_task_custom_attr = {
"url",
Expand All @@ -68,7 +94,7 @@ def __init__(
name: str,
url: str,
http_method: str | None = HttpMethod.GET,
http_params: str | None = None,
http_params: dict | None = None,
http_check_condition: str | None = HttpCheckCondition.STATUS_CODE_DEFAULT,
condition: str | None = None,
connect_timeout: int | None = 60000,
Expand All @@ -82,8 +108,16 @@ def __init__(
raise PyDSParamException(
"Parameter http_method %s not support.", http_method
)

if isinstance(http_params, list):
warnings.warn(
"The `http_params` parameter currently accepts a dictionary instead of a list. Your parameter is being ignored.",
DeprecationWarning,
)

self.http_method = http_method
self.http_params = http_params or []
self._http_params = http_params

if not hasattr(HttpCheckCondition, http_check_condition):
raise PyDSParamException(
"Parameter http_check_condition %s not support.", http_check_condition
Expand All @@ -99,3 +133,12 @@ def __init__(
self.condition = condition
self.connect_timeout = connect_timeout
self.socket_timeout = socket_timeout

@property
def http_params(self):
"""Property to convert http_params using ParameterHelper when accessed."""
return (
ParameterHelper.convert_params(self._http_params, direction=Direction.IN)
if self._http_params
else []
)
55 changes: 55 additions & 0 deletions tests/tasks/test_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

"""Test Task HTTP."""

import warnings
from unittest.mock import patch

import pytest
Expand Down Expand Up @@ -126,3 +127,57 @@ def test_http_get_define():
):
http = Http(name, url)
assert http.task_params == expect_task_params


@patch(
"pydolphinscheduler.core.task.Task.gen_code_and_version",
return_value=(123, 1),
)
def test_http_params(mock_code):
"""Test the transformation and conversion of http_params."""
http_params_dict = {"prop1": "value1", "prop2": 135, "prop3": "value3"}

http_instance = Http(
name="test_http",
url="http://www.example.com",
http_method="GET",
http_params=http_params_dict,
)
expect_transformation = [
{"prop": "prop1", "direct": "IN", "type": "VARCHAR", "value": "value1"},
{"prop": "prop2", "direct": "IN", "type": "INTEGER", "value": 135},
{"prop": "prop3", "direct": "IN", "type": "VARCHAR", "value": "value3"},
]

assert isinstance(http_instance.http_params, list)
assert len(http_instance.http_params) == len(http_params_dict)
assert http_instance.http_params == expect_transformation


def test_http_params_deprecation_warning():
"""Test deprecation warning when user passes list to http_params."""
code = 123
version = 1
name = "test_http_params_deprecation_warning"
http_params_list = [
{"prop": "abc", "httpParametersType": "PARAMETER", "value": "def"}
]

with patch(
"pydolphinscheduler.core.task.Task.gen_code_and_version",
return_value=(code, version),
):
with warnings.catch_warnings(record=True) as w:
Http(
name=name,
url="http://www.example.com",
http_method="GET",
http_params=http_params_list,
)

# Check if a deprecation warning is raised
assert len(w) == 1
assert issubclass(w[-1].category, DeprecationWarning)
assert "The `http_params` parameter currently accepts a dictionary" in str(
w[-1].message
)

0 comments on commit 0f120f6

Please sign in to comment.