Source Confluence: certificate to Beta

airbyte-integrations/connectors/source-confluence/Dockerfile

@@ -34,5 +34,5 @@ COPY source_confluence ./source_confluence
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-confluence

airbyte-integrations/connectors/source-confluence/acceptance-test-config.yml

@@ -1,20 +1,29 @@
 # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference)
 # for more information about how to configure these tests
 connector_image: airbyte/source-confluence:dev
-tests:
+acceptance_tests:
   spec:
+    tests:
     - spec_path: "source_confluence/spec.json"
   connection:
+    tests:
     - config_path: "secrets/config.json"
       status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:
+    tests:
     - config_path: "secrets/config.json"
   basic_read:
+    tests:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
-      empty_streams: []
+      expect_records:
+        path: "integration_tests/expected_records.jsonl"
   full_refresh:
+    tests:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+      ignored_fields:
+        pages:
+          - name: "body/view"

airbyte-integrations/connectors/source-confluence/integration_tests/configured_catalog.json

@@ -1,5 +1,15 @@
 {
   "streams": [
+    {
+      "stream": {
+        "name": "audit",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["author"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "pages",

airbyte-integrations/connectors/source-confluence/source_confluence/schemas/pages.json

@@ -159,6 +159,61 @@
         }
       }
     },
+    "body": {
+      "type": "object",
+			"properties": {
+        "storage": {
+          "type": "object",
+					"properties": {
+						"value": {
+              "type": "string"
+            },
+            "representation": {
+              "type": "string"
+            },
+            "embeddedContent": {
+              "type": "array"
+            },
+            "_expandable": {
+              "type": "object",
+							"properties": {
+                "content": {
+                  "type": "string"
+                }
+              }
+            }
+          }
+        },
+        "view": {
+          "type": "object",
+					"properties": {
+						"value": {
+              "type": "string"
+            },
+            "representation": {
+              "type": "string"
+            },
+            "_expandable": {
+              "type": "object",
+							"properties": {
+								"webresource": {
+                  "type": "string"
+                }
+              }
+            },
+            "embeddedContent": {
+              "type": "string"
+            },
+            "mediaToken": {
+              "type": "string"
+            },
+            "content": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    },
     "restrictions": {
       "type": "object",
       "properties": {

airbyte-integrations/connectors/source-confluence/source_confluence/source.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
 
-
+import logging
 from abc import ABC
 from base64 import b64encode
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple
@@ -14,6 +14,8 @@
 from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 
+logger = logging.getLogger("airbyte")
+
 
 # Basic full refresh stream
 class ConfluenceStream(HttpStream, ABC):
@@ -66,6 +68,9 @@ class BaseContentStream(ConfluenceStream, ABC):
         "restrictions.read.restrictions.user",
         "version",
         "descendants.comment",
+        "body",
+        "body.storage",
+        "body.view",
     ]
     limit = 25
     content_type = None
@@ -143,4 +148,12 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         auth = HttpBasicAuthenticator(config["email"], config["api_token"], auth_method="Basic")
         config["authenticator"] = auth
-        return [Pages(config), BlogPosts(config), Space(config), Group(config), Audit(config)]
+        # stream Audit requires Premium or Standard Plan
+        url = f"https://{config['domain_name']}/wiki/rest/api/audit?limit=1"
+        try:
+            response = requests.get(url, headers=auth.get_auth_header())
+            response.raise_for_status()
+            return [Pages(config), BlogPosts(config), Space(config), Group(config), Audit(config)]
+        except requests.exceptions.RequestException as e:
+            logger.warning(f"An exception occurred while trying to access Audit stream: {str(e)}. Skipping this stream.")
+            return [Pages(config), BlogPosts(config), Space(config), Group(config)]

airbyte-integrations/connectors/source-confluence/unit_tests/test_source.py

@@ -24,9 +24,20 @@ def test_check_connection(config):
     assert source.check_connection(logger_mock, config) == (True, None)
 
 
-def test_streams_count(mocker):
+def test_check_connection_failed(config):
     source = SourceConfluence()
-    config_mock = MagicMock()
-    streams = source.streams(config_mock)
+    logger_mock = MagicMock()
+    assert source.check_connection(logger_mock, config)[0] is False
+
+
+@responses.activate
+def test_streams_count(config):
+    responses.add(
+        responses.GET,
+        "https://example.atlassian.net/wiki/rest/api/audit",
+        status=200,
+    )
+    source = SourceConfluence()
+    streams = source.streams(config)
     expected_streams_number = 5
     assert len(streams) == expected_streams_number

airbyte-integrations/connectors/source-confluence/unit_tests/test_streams.py

@@ -0,0 +1,131 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from unittest.mock import MagicMock
+
+import pytest
+import requests
+from source_confluence.source import Audit, BaseContentStream, BlogPosts, ConfluenceStream, Group, Pages, Space
+
+
+class TestsConfluenceStream:
+    confluence_stream_class = ConfluenceStream({"authenticator": MagicMock(), "domain_name": "test"})
+
+    def tests_url_base(self):
+        assert self.confluence_stream_class.url_base == "https://test/wiki/rest/api/"
+
+    def test_primary_key(self):
+        assert self.confluence_stream_class.primary_key == "id"
+
+    def test_limit(self):
+        assert self.confluence_stream_class.limit == 50
+
+    def test_start(self):
+        assert self.confluence_stream_class.start == 0
+
+    def test_expand(self):
+        assert self.confluence_stream_class.expand == []
+
+    def test_next_page_token(self, requests_mock):
+        url = "https://test.atlassian.net/wiki/rest/api/space"
+        requests_mock.get(url, status_code=200, json={"_links": {"next": "test link"}})
+        response = requests.get(url)
+        assert self.confluence_stream_class.next_page_token(response=response) == {"start": 50}
+
+    @pytest.mark.parametrize(
+        ("stream_state", "stream_slice", "next_page_token", "expected"),
+        [
+            ({}, {}, {}, {'limit': 50, 'expand': ''}),
+        ],
+    )
+    def test_request_params(self, stream_state, stream_slice, next_page_token, expected):
+        assert self.confluence_stream_class.request_params(stream_state, stream_slice, next_page_token) == expected
+
+    def test_parse_response(self, requests_mock):
+        url = "https://test.atlassian.net/wiki/rest/api/space"
+        requests_mock.get(url, status_code=200, json={"results": ["test", "test1", "test3"]})
+        response = requests.get(url)
+        assert list(self.confluence_stream_class.parse_response(response=response)) == ['test', 'test1', 'test3']
+
+
+class TestBaseContentStream:
+    base_content_stream_class = BaseContentStream({"authenticator": MagicMock(), "domain_name": "test"})
+
+    def test_path(self):
+        assert self.base_content_stream_class.path({}, {}, {}) == "content"
+
+    def test_expand(self):
+        assert self.base_content_stream_class.expand == ["history",
+                                                         "history.lastUpdated",
+                                                         "history.previousVersion",
+                                                         "history.contributors",
+                                                         "restrictions.read.restrictions.user",
+                                                         "version",
+                                                         "descendants.comment",
+                                                         "body",
+                                                         "body.storage",
+                                                         "body.view",
+                                                         ]
+
+    def test_limit(self):
+        assert self.base_content_stream_class.limit == 25
+
+    def test_content_type(self):
+        assert self.base_content_stream_class.content_type is None
+
+    @pytest.mark.parametrize(
+        ("stream_state", "stream_slice", "next_page_token", "expected"),
+        [
+            ({}, {}, {}, {'limit': 25,
+                          'expand': 'history,history.lastUpdated,history.previousVersion,history.contributors,restrictions.'
+                                    'read.restrictions.user,version,descendants.comment,body,body.storage,body.view',
+                          'type': None}),
+        ],
+    )
+    def test_request_params(self, stream_state, stream_slice, next_page_token, expected):
+        assert self.base_content_stream_class.request_params(stream_state, stream_slice, next_page_token) == expected
+
+
+class TestPages:
+    pages_class = Pages({"authenticator": MagicMock(), "domain_name": "test"})
+
+    def test_content_type(self):
+        assert self.pages_class.content_type == "page"
+
+
+class TestBlogPosts:
+    blog_posts_class = BlogPosts({"authenticator": MagicMock(), "domain_name": "test"})
+
+    def test_content_type(self):
+        assert self.blog_posts_class.content_type == "blogpost"
+
+
+class TestSpace:
+    space_class = Space({"authenticator": MagicMock(), "domain_name": "test"})
+
+    def test_api_name(self):
+        assert self.space_class.api_name == "space"
+
+    def test_expand(self):
+        assert self.space_class.expand == ["permissions", "icon", "description.plain", "description.view"]
+
+
+class TestGroup:
+    group_class = Group({"authenticator": MagicMock(), "domain_name": "test"})
+
+    def test_api_name(self):
+        assert self.group_class.api_name == "group"
+
+
+class TestAudit:
+    audit_class = Audit({"authenticator": MagicMock(), "domain_name": "test"})
+
+    def test_api_name(self):
+        assert self.audit_class.api_name == "audit"
+
+    def test_primary_key(self):
+        assert self.audit_class.primary_key == "author"
+
+    def test_limit(self):
+        assert self.audit_class.limit == 1000

docs/integrations/sources/confluence.md

@@ -14,7 +14,7 @@ This page contains the setup guide and reference information for the Confluence
 2. Click **Sources** and then click **+ New source**. 
 3. On the Set up the source page, select **Confluence** from the Source type dropdown.
 4. Enter a name for your source.
-5. For **API Tokene** follow the Jira confluence for generating an  [API Token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/)
+5. For **API Token** follow the Jira confluence for generating an  [API Token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/)
 6. For **Domain name** enter your Confluence domain name.
 7. For **Email** enter your Confluence login email.
 
@@ -35,7 +35,9 @@ This page contains the setup guide and reference information for the Confluence
 * [Space](https://developer.atlassian.com/cloud/confluence/rest/api-group-space/#api-wiki-rest-api-space-get)
 * [Group](https://developer.atlassian.com/cloud/confluence/rest/api-group-group/#api-wiki-rest-api-group-get)
 * [Audit](https://developer.atlassian.com/cloud/confluence/rest/api-group-audit/#api-wiki-rest-api-audit-get)
-
+:::warning
+Stream Audit requires Standard or Premium plan.
+:::
 ## Data type map
 The [Confluence API](https://developer.atlassian.com/cloud/confluence/rest/intro/#about) uses the same [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions happen as part of this source.
 
@@ -45,8 +47,9 @@ The Confluence connector should not run into Confluence API limitations under no
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                              |
-|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------|
-| 0.1.2   | 2023-03-06 | [23775](https://github.com/airbytehq/airbyte/pull/23775) | Set additionalProperties: true, update docs and spec |
-| 0.1.1   | 2022-01-31 | [9831](https://github.com/airbytehq/airbyte/pull/9831)   | Fix: Spec was not pushed to cache                    |
-| 0.1.0   | 2021-11-05 | [7241](https://github.com/airbytehq/airbyte/pull/7241)   | 🎉 New Source: Confluence                            |
+| Version | Date       | Pull Request                                             | Subject                                                        |
+|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------|
+| 0.1.3   | 2023-03-13 | [23988](https://github.com/airbytehq/airbyte/pull/23988) | Add view and storage to pages body, add check for stream Audit |
+| 0.1.2   | 2023-03-06 | [23775](https://github.com/airbytehq/airbyte/pull/23775) | Set additionalProperties: true, update docs and spec           |
+| 0.1.1   | 2022-01-31 | [9831](https://github.com/airbytehq/airbyte/pull/9831)   | Fix: Spec was not pushed to cache                              |
+| 0.1.0   | 2021-11-05 | [7241](https://github.com/airbytehq/airbyte/pull/7241)   | 🎉 New Source: Confluence                                      |