Skip to content

feat: restore os provider on core library #64764

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 3 commits into from
Apr 10, 2026
+26 −23
Merged

feat: restore os provider on core library #64764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e523e89
add changes for os provider
Owen-CH-Leung Apr 6, 2026
1eba4e9
bump pyproject.toml provider lib version
Owen-CH-Leung Apr 8, 2026
b1b39ac
Fix failing CI
Owen-CH-Leung Apr 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 23 additions & 21 deletions airflow-core/src/airflow/config_templates/airflow_local_settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,34 +334,36 @@ def _default_conn_name_from(mod_path, hook_name):
)

elif OPENSEARCH_HOST:
OPENSEARCH_END_OF_LOG_MARK: str = conf.get_mandatory_value("opensearch", "END_OF_LOG_MARK")
OPENSEARCH_PORT: str = conf.get_mandatory_value("opensearch", "PORT")
from airflow.providers.opensearch.log.os_task_handler import OpensearchRemoteLogIO

OPENSEARCH_PORT = conf.getint("opensearch", "PORT", fallback=9200)
OPENSEARCH_USERNAME: str = conf.get_mandatory_value("opensearch", "USERNAME")
OPENSEARCH_PASSWORD: str = conf.get_mandatory_value("opensearch", "PASSWORD")
OPENSEARCH_WRITE_STDOUT: bool = conf.getboolean("opensearch", "WRITE_STDOUT")
OPENSEARCH_WRITE_TO_OS: bool = conf.getboolean("opensearch", "WRITE_TO_OS")
OPENSEARCH_JSON_FORMAT: bool = conf.getboolean("opensearch", "JSON_FORMAT")
OPENSEARCH_JSON_FIELDS: str = conf.get_mandatory_value("opensearch", "JSON_FIELDS")
OPENSEARCH_TARGET_INDEX: str = conf.get_mandatory_value("opensearch", "TARGET_INDEX")
OPENSEARCH_HOST_FIELD: str = conf.get_mandatory_value("opensearch", "HOST_FIELD")
OPENSEARCH_OFFSET_FIELD: str = conf.get_mandatory_value("opensearch", "OFFSET_FIELD")
OPENSEARCH_LOG_ID_TEMPLATE: str = conf.get("opensearch", "LOG_ID_TEMPLATE", fallback="") or (
"{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}"
)

OPENSEARCH_REMOTE_HANDLERS: dict[str, dict[str, str | bool | None]] = {
"task": {
"class": "airflow.providers.opensearch.log.os_task_handler.OpensearchTaskHandler",
"formatter": "airflow",
"base_log_folder": BASE_LOG_FOLDER,
"end_of_log_mark": OPENSEARCH_END_OF_LOG_MARK,
"host": OPENSEARCH_HOST,
"port": OPENSEARCH_PORT,
"username": OPENSEARCH_USERNAME,
"password": OPENSEARCH_PASSWORD,
"write_stdout": OPENSEARCH_WRITE_STDOUT,
"json_format": OPENSEARCH_JSON_FORMAT,
"json_fields": OPENSEARCH_JSON_FIELDS,
"host_field": OPENSEARCH_HOST_FIELD,
"offset_field": OPENSEARCH_OFFSET_FIELD,
},
}
DEFAULT_LOGGING_CONFIG["handlers"].update(OPENSEARCH_REMOTE_HANDLERS)
REMOTE_TASK_LOG = OpensearchRemoteLogIO(
host=OPENSEARCH_HOST,
port=OPENSEARCH_PORT,
username=OPENSEARCH_USERNAME,
password=OPENSEARCH_PASSWORD,
target_index=OPENSEARCH_TARGET_INDEX,
write_stdout=OPENSEARCH_WRITE_STDOUT,
write_to_opensearch=OPENSEARCH_WRITE_TO_OS,
offset_field=OPENSEARCH_OFFSET_FIELD,
host_field=OPENSEARCH_HOST_FIELD,
base_log_folder=BASE_LOG_FOLDER,
delete_local_copy=delete_local_copy,
json_format=OPENSEARCH_JSON_FORMAT,
log_id_template=OPENSEARCH_LOG_ID_TEMPLATE,
)
else:
raise AirflowException(
"Incorrect remote log configuration. Please check the configuration of option 'host' in "
Expand Down
4 changes: 2 additions & 2 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ apache-airflow = "airflow.__main__:main"
"apache-airflow-providers-openlineage>=2.3.0" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
]
"opensearch" = [
"apache-airflow-providers-opensearch>=1.5.0"
"apache-airflow-providers-opensearch>=1.9.0" # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
]
"opsgenie" = [
"apache-airflow-providers-opsgenie>=5.8.0"
Expand Down Expand Up @@ -464,7 +464,7 @@ apache-airflow = "airflow.__main__:main"
"apache-airflow-providers-openai>=1.5.0",
"apache-airflow-providers-openfaas>=3.7.0",
"apache-airflow-providers-openlineage>=2.3.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
"apache-airflow-providers-opensearch>=1.5.0",
"apache-airflow-providers-opensearch>=1.9.0", # Set from MIN_VERSION_OVERRIDE in update_airflow_pyproject_toml.py
"apache-airflow-providers-opsgenie>=5.8.0",
"apache-airflow-providers-oracle>=3.12.0",
"apache-airflow-providers-pagerduty>=3.8.1",
Expand Down
1 change: 1 addition & 0 deletions scripts/ci/prek/update_airflow_pyproject_toml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@
"git": parse_version("0.0.2"),
"common.messaging": parse_version("2.0.0"),
"elasticsearch": parse_version("6.5.0"),
"opensearch": parse_version("1.9.0"),
}


Expand Down
Loading