Airflow Api-Server 3.1 fails to start after upgrading FAB provider to apache-airflow-providers-fab==3.0.0 #57026
Description
Apache Airflow Provider(s)
fab
Versions of Apache Airflow Providers
apache-airflow==3.1.0
apache-airflow-core==3.1.0
apache-airflow-providers-amazon==9.15.0
apache-airflow-providers-celery==3.12.4
apache-airflow-providers-common-compat==1.7.4
apache-airflow-providers-common-io==1.6.3
apache-airflow-providers-common-sql==1.28.1
apache-airflow-providers-docker==4.4.3
apache-airflow-providers-fab==3.0.0
apache-airflow-providers-hashicorp==4.3.2
apache-airflow-providers-http==5.3.4
apache-airflow-providers-smtp==2.2.1
apache-airflow-providers-standard==1.8.0
apache-airflow-providers-trino==6.3.3
apache-airflow-task-sdk==1.1.0
Apache Airflow version
3.1
Operating System
Debian GNU/Linux 12
Deployment
Docker-Compose
Deployment details
No response
What happened
After upgrading apache-airflow-providers-fab to 3.0.0, Airflow Api-Server refuses to start and fails with error
[2025-10-22T05:45:07.641+0000] {cli_parser.py:81} WARNING - cannot load CLI commands from auth manager: The package `apache-airflow-providers-fab:3.0.0` needs Apache Airflow 3.0.2+
[2025-10-22T05:45:07.645+0000] {cli_parser.py:82} WARNING - Auth manager is not configured and api-server will not be able to start.
[2025-10-22T05:45:07.650+0000] {cli_parser.py:85} ERROR - The package `apache-airflow-providers-fab:3.0.0` needs Apache Airflow 3.0.2+
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.12/site-packages/airflow/cli/cli_parser.py", line 78, in <module>
auth_mgr = get_auth_manager_cls()
^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/airflow/api_fastapi/app.py", line 125, in get_auth_manager_cls
auth_manager_cls = conf.getimport(section="core", key="auth_manager")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/airflow/configuration.py", line 1229, in getimport
return import_string(full_qualified_path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/airflow/.local/lib/python3.12/site-packages/airflow/utils/module_loading.py", line 39, in import_string
module = import_module(module_path)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/python/lib/python3.12/importlib/__init__.py", line 90, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
File "<frozen importlib._bootstrap>", line 1310, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
File "<frozen importlib._bootstrap>", line 1310, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 999, in exec_module
File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
File "/home/airflow/.local/lib/python3.12/site-packages/airflow/providers/fab/__init__.py", line 37, in <module>
raise RuntimeError(
RuntimeError: The package `apache-airflow-providers-fab:3.0.0` needs Apache Airflow 3.0.2+
It clearly can't see Airflow 3.1, which is newer than 3.0.2
What you think should happen instead
Successful api-server start with fab provider 3.0
How to reproduce
Have Airflow 3.1 running and run pip install apache-airflow-providers-fab==3.0.0 --upgrade. Reboot api-server and the error will appear
Anything else
If needed, my webserver_config.py file
from __future__ import annotations
import os
import logging
import jwt
import requests
from base64 import b64decode
from airflow import settings
from cryptography.hazmat.primitives import serialization
from flask_appbuilder.security.manager import AUTH_OAUTH
from airflow.providers.fab.auth_manager.security_manager.override import (
FabAirflowSecurityManagerOverride,
)
basedir = os.path.abspath(os.path.dirname(__file__))
log = logging.getLogger(__name__)
# Flask-WTF flag for CSRF
WTF_CSRF_ENABLED = True
WTF_CSRF_TIME_LIMIT = None
# ----------------------------------------------------
# AUTHENTICATION CONFIG
# ----------------------------------------------------
# For details on how to set up each of the following, see
# http://flask-appbuilder.readthedocs.io/en/latest/security.html#authentication-methods
# The authentication type
# AUTH_OID : Is for OpenID
# AUTH_DB : For Airflow DB (Default)
# AUTH_LDAP : Is for LDAP
# AUTH_REMOTE_USER : Is for using REMOTE_USER from web server
# AUTH_OAUTH : Is for OAuth <- Keycloak uses OAuth
AUTH_TYPE = AUTH_OAUTH
# Required for Airflow to recognize users on their first time authenticating with Keycloak
AUTH_USER_REGISTRATION = True
# The default role if the user hasn't been assigned one in Keycloak
AUTH_USER_REGISTRATION_ROLE = "Viewer"
#AUTH_ROLES_SYNC_AT_LOGIN = False
# airflow_xyz is the arbitrarily-chosen name of the role defined in Keycloak.
# Whereas "Admin", "Op", etc. are defined by Airflow: https://airflow.apache.org/docs/apache-airflow/stable/security/access-control.html
AUTH_ROLES_MAPPING = {
"airflow_admin": ["Admin"],
"airflow_op": ["Op"],
"airflow_user": ["User"],
"airflow_viewer": ["Viewer"],
"airflow_public": ["Public"],
# add custom roles here
}
#AUTH_ROLES_SYNC_AT_LOGIN = True
# Keycloak variables
PROVIDER_NAME = "keycloak"
CLIENT_ID = "airflow"
CLIENT_SECRET = ""
OIDC_ISSUER = "https://idp.test.com/realms/internal"
OIDC_BASE_URL = "{oidc_issuer}/protocol/openid-connect".format(oidc_issuer=OIDC_ISSUER)
OIDC_TOKEN_URL = "{oidc_base_url}/token".format(oidc_base_url=OIDC_BASE_URL)
OIDC_AUTH_URL = "{oidc_base_url}/auth".format(oidc_base_url=OIDC_BASE_URL)
OAUTH_PROVIDERS = [
{
"name": PROVIDER_NAME,
"token_key": "access_token",
"icon": "fa-circle-o",
"remote_app": {
"api_base_url": f"{OIDC_BASE_URL}/",
"access_token_url": OIDC_TOKEN_URL,
"authorize_url": OIDC_AUTH_URL,
"request_token_url": None,
"client_id": CLIENT_ID,
"client_secret": CLIENT_SECRET,
"client_kwargs": {"scope": "email profile"},
},
}
]
# Get public key of your Keycloak instance
req = requests.get(OIDC_ISSUER)
key_der_base64 = req.json()["public_key"]
key_der = b64decode(key_der_base64.encode())
public_key = serialization.load_der_public_key(key_der)
class CustomSecurityManager(FabAirflowSecurityManagerOverride):
def get_oauth_user_info(self, provider, response):
if provider == "keycloak":
token = response["access_token"]
me = jwt.decode(token, public_key, algorithms=["HS256", "RS256"], audience=CLIENT_ID)
login_user = me.get("preferred_username")
# Check if user exists in Airflow database
session = settings.Session()
airflow_users = session.execute("SELECT username FROM ab_user").fetchall()
usernames = [user[0] for user in airflow_users]
if login_user not in usernames:
# If user does not exist, create a new user with the default role
groups = ["Viewer"]
userinfo = {
"username": login_user,
"email": me.get("email"),
"first_name": me.get("given_name"),
"last_name": me.get("family_name"),
"role_keys": groups,
}
else:
# If user exists, fetch their roles from the Airflow database
user_roles = self.get_user_roles_from_db(session, login_user)
userinfo = {
"username": login_user,
"email": me.get("email"),
"first_name": me.get("given_name"),
"last_name": me.get("family_name"),
"role_keys": user_roles,
}
session.close()
log.info("user info: {0}".format(userinfo))
return userinfo
else:
return {}
def get_user_roles_from_db(self, session, username):
# Query the Airflow database to get the roles for the user
roles_query = """
SELECT r.name
FROM ab_user u
JOIN ab_user_role ur ON u.id = ur.user_id
JOIN ab_role r ON ur.role_id = r.id
WHERE u.username = :username
"""
roles = session.execute(roles_query, {"username": username}).fetchall()
return [role[0] for role in roles]
SECURITY_MANAGER_CLASS = CustomSecurityManager
Are you willing to submit PR?
- Yes I am willing to submit a PR!
Code of Conduct
- I agree to follow this project's Code of Conduct