Adding GitHub provider (#21076)

.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml

@@ -51,6 +51,7 @@ body:
         - exasol
         - facebook
         - ftp
+        - github
         - google
         - grpc
         - hashicorp

CONTRIBUTING.rst

@@ -609,13 +609,13 @@ apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, a
 apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery,
 cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel,
 devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol,
-facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive,
-http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
-microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas,
-opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds,
-qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack,
-snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, virtualenv, webhdfs,
-winrm, yandex, zendesk
+facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs,
+hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb,
+microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc,
+openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto,
+qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity,
+slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, virtualenv,
+webhdfs, winrm, yandex, zendesk
 
   .. END EXTRAS HERE
 

INSTALL

@@ -100,13 +100,13 @@ apache.druid, apache.hdfs, apache.hive, apache.kylin, apache.livy, apache.pig, a
 apache.spark, apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery,
 cgroups, cloudant, cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel,
 devel_all, devel_ci, devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol,
-facebook, ftp, gcp, gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive,
-http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure,
-microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas,
-opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto, qds,
-qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack,
-snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, virtualenv, webhdfs,
-winrm, yandex, zendesk
+facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs,
+hive, http, imap, influxdb, jdbc, jenkins, jira, kerberos, kubernetes, ldap, leveldb,
+microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc,
+openfaas, opsgenie, oracle, pagerduty, pandas, papermill, password, pinot, plexus, postgres, presto,
+qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid, sentry, sftp, singularity,
+slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino, vertica, virtualenv,
+webhdfs, winrm, yandex, zendesk
 
 # END EXTRAS HERE
 

airflow/providers/github/CHANGELOG.rst

@@ -0,0 +1,25 @@
+
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+Changelog
+---------
+1.0.0
+.....
+
+Initial version of the provider.

airflow/providers/github/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/github/example_dags/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/github/example_dags/example_github.py

@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import logging
+from datetime import datetime
+from typing import Any, Optional
+
+from github import GithubException
+
+from airflow import AirflowException
+from airflow.models.dag import DAG
+from airflow.providers.github.operators.github import GithubOperator
+from airflow.providers.github.sensors.github import GithubSensor, GithubTagSensor
+
+dag = DAG(
+    'example_github_operator',
+    start_date=datetime(2021, 1, 1),
+    tags=['example'],
+    catchup=False,
+)
+
+# [START howto_tag_sensor_github]
+
+tag_sensor = GithubTagSensor(
+    task_id='example_tag_sensor',
+    tag_name='v1.0',
+    repository_name="apache/airflow",
+    timeout=60,
+    poke_interval=10,
+    dag=dag,
+)
+
+
+# [END howto_tag_sensor_github]
+
+# [START howto_sensor_github]
+
+
+def tag_checker(repo: Any, tag_name: str) -> Optional[bool]:
+    result = None
+    try:
+        if repo is not None and tag_name is not None:
+            all_tags = [x.name for x in repo.get_tags()]
+            result = tag_name in all_tags
+
+    except GithubException as github_error:
+        raise AirflowException(f"Failed to execute GithubSensor, error: {str(github_error)}")
+    except Exception as e:
+        raise AirflowException(f"Github operator error: {str(e)}")
+    return result
+
+
+github_sensor = GithubSensor(
+    task_id='example_sensor',
+    method_name="get_repo",
+    method_params={'full_name_or_id': "apache/airflow"},
+    result_processor=lambda repo: tag_checker(repo, 'v1.0'),
+    timeout=60,
+    poke_interval=10,
+    dag=dag,
+)
+
+# [END howto_sensor_github]
+
+
+# [START howto_operator_list_repos_github]
+
+github_list_repos = GithubOperator(
+    task_id='github_list_repos',
+    github_method="get_user",
+    github_method_args={},
+    result_processor=lambda user: logging.info(list(user.get_repos())),
+    dag=dag,
+)
+
+# [END howto_operator_list_repos_github]
+
+# [START howto_operator_list_tags_github]
+
+list_repo_tags = GithubOperator(
+    task_id='list_repo_tags',
+    github_method="get_repo",
+    github_method_args={'full_name_or_id': 'apache/airflow'},
+    result_processor=lambda repo: logging.info(list(repo.get_tags())),
+    dag=dag,
+)
+
+# [END howto_operator_list_tags_github]

airflow/providers/github/hooks/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/github/hooks/github.py

@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""This module allows to connect to a Github."""
+from typing import Dict, Optional
+
+from github import Github as GithubClient
+
+from airflow.hooks.base import BaseHook
+
+
+class GithubHook(BaseHook):
+    """
+    Interact with Github.
+
+    Performs a connection to GitHub and retrieves client.
+
+    :param github_conn_id: Reference to :ref:`GitHub connection id <howto/connection:github>`.
+    :type github_conn_id: str
+    """
+
+    conn_name_attr = 'github_conn_id'
+    default_conn_name = 'github_default'
+    conn_type = 'github'
+    hook_name = 'Github'
+
+    def __init__(self, github_conn_id: str = default_conn_name, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.github_conn_id = github_conn_id
+        self.client: Optional[GithubClient] = None
+        self.get_conn()
+
+    def get_conn(self) -> GithubClient:
+        """
+        Function that initiates a new GitHub connection
+        with token and hostname ( for GitHub Enterprise )
+        """
+        if self.client is not None:
+            return self.client
+
+        conn = self.get_connection(self.github_conn_id)
+        access_token = conn.password
+        host = conn.host
+
+        if not host:
+            self.client = GithubClient(login_or_token=access_token)
+        else:
+            self.client = GithubClient(login_or_token=access_token, base_url=host)
+
+        return self.client
+
+    @staticmethod
+    def get_ui_field_behaviour() -> Dict:
+        """Returns custom field behaviour"""
+        return {
+            "hidden_fields": ['schema', 'port', 'login', 'extra'],
+            "relabeling": {
+                'host': 'Github Enterprise Url (Optional)',
+                'password': 'Github Access Token',
+            },
+            "placeholders": {
+                'host': 'https://{hostname}/api/v3 (for Github Enterprise Connection)',
+                'password': 'token credentials auth',
+            },
+        }

airflow/providers/github/operators/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.