Merge branch 'main' into fixup-x-pack-test_serverless-functional-test…

…_suites-observability-cases-list_view
elastic · Jun 3, 2024 · 8dac7c9 · 8dac7c9
2 parents dda6f81 + 838be8c
commit 8dac7c9
Show file tree

Hide file tree

Showing 26 changed files with 810 additions and 755 deletions.
diff --git a/docs/user/monitoring/elasticsearch-details.asciidoc b/docs/user/monitoring/elasticsearch-details.asciidoc
@@ -108,16 +108,39 @@ model, the number of forecasts, and the node that runs the job.
 == CCR
 
 To view {ccr} metrics, click **CCR**. For each follower index on the cluster, it 
-shows information such as the leader index, an indication of how much the
-follower index is lagging behind the leader index, the last fetch time, the
-number of operations synced, and error messages. If you select a follower index,
-you can view the same information for each shard.
+shows the following information:
+
+- **Index**: The name of the follower index.
+- **Follows**: The name of the leader index.
+- **Alerts**: Any read exceptions that have been triggered for the index or its 
+shards.
+- **Sync Lag (ops)**: How many operations the follower index is lagging behind the 
+leader index. 
++
+This is calculated by finding the difference between the minimum and maximum operation 
+sequence number on the leader (`leader_max_seq_no`) and the difference between the minimum 
+and maximum global sequence number checkpoint on the follower (`follower_global_checkpoint`) 
+for each shard over the selected time period. The difference in `follower_global_checkpoint` 
+is subtracted from the difference in `leader_max_seq_no` for each shard, and the highest result 
+across all shards is displayed.
+- **Last fetch time**: The time elapsed since the last successful fetch from the leader index. 
+Represents the longest time elapsed across all of the shards in the follower index.
+- **Ops synced**: The number of operations indexed (replicated) into the follower index from 
+the leader index in the selected time period. 
++
+This metric is a sum of the number of operations indexed across all shards over the selected 
+time period.
+- **Error**: Any exceptions returned for the most recent document in the selected time period.
+
+If you select a follower index, you can view the same information for each shard. 
+For more information on the properties used to calculate these metrics, refer to the 
+{ref}/ccr-get-follow-stats.html[get follower stats API] documentation.
 
 If you select a shard, you can see graphs for the fetch and operation delays. 
-You can also see advanced information, which contains the results from the 
+You can also see advanced information, which contains additional stats from the 
 {ref}/ccr-get-follow-stats.html[get follower stats API].
 
-For more information, refer to {ref}/xpack-ccr.html[{ccr-cap}].
+Learn more about {ref}/xpack-ccr.html[{ccr-cap}].
 
 [float]
 [[logs-monitor-page]]

diff --git a/packages/kbn-search-connectors/lib/start_sync.test.ts b/packages/kbn-search-connectors/lib/start_sync.test.ts
@@ -8,235 +8,97 @@
 
 import { ElasticsearchClient } from '@kbn/core/server';
 
-import { CONNECTORS_INDEX, CURRENT_CONNECTORS_JOB_INDEX } from '..';
-import { SyncJobType, SyncStatus, TriggerMethod } from '../types/connectors';
-import { CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX } from '..';
+import { errors } from '@elastic/elasticsearch';
+
+import { SyncJobType } from '../types/connectors';
 
 import { startConnectorSync } from './start_sync';
 
 describe('startSync lib function', () => {
   const mockClient = {
-    get: jest.fn(),
-    index: jest.fn(),
-    update: jest.fn(),
+    transport: {
+      request: jest.fn(),
+    },
   };
 
   beforeEach(() => {
     jest.clearAllMocks();
   });
 
   it('should start a full sync', async () => {
-    mockClient.get.mockImplementation(() => {
-      return Promise.resolve({
-        _id: 'connectorId',
-        _source: {
-          api_key_id: null,
-          configuration: {},
-          created_at: null,
-          custom_scheduling: {},
-          error: null,
-          index_name: 'index_name',
-          language: null,
-          last_access_control_sync_error: null,
-          last_access_control_sync_scheduled_at: null,
-          last_access_control_sync_status: null,
-          last_seen: null,
-          last_sync_error: null,
-          last_sync_scheduled_at: null,
-          last_sync_status: null,
-          last_synced: null,
-          scheduling: { enabled: true, interval: '1 2 3 4 5' },
-          service_type: null,
-          status: 'not connected',
-          sync_now: false,
-        },
-        index: CONNECTORS_INDEX,
-      });
-    });
-    mockClient.index.mockImplementation(() => ({ _id: 'fakeId' }));
+    mockClient.transport.request.mockImplementation(() => ({ id: '12345' }));
 
     await expect(
       startConnectorSync(mockClient as unknown as ElasticsearchClient, {
         connectorId: 'connectorId',
         jobType: SyncJobType.FULL,
       })
-    ).resolves.toEqual({ _id: 'fakeId' });
-    expect(mockClient.index).toHaveBeenCalledWith({
-      document: {
-        cancelation_requested_at: null,
-        canceled_at: null,
-        completed_at: null,
-        connector: {
-          configuration: {},
-          filtering: null,
-          id: 'connectorId',
-          index_name: 'index_name',
-          language: null,
-          pipeline: null,
-          service_type: null,
-        },
-        created_at: expect.any(String),
-        deleted_document_count: 0,
-        error: null,
-        indexed_document_count: 0,
-        indexed_document_volume: 0,
-        job_type: SyncJobType.FULL,
-        last_seen: null,
-        metadata: {},
-        started_at: null,
-        status: SyncStatus.PENDING,
-        total_document_count: null,
-        trigger_method: TriggerMethod.ON_DEMAND,
-        worker_hostname: null,
+    ).resolves.toEqual({ id: '12345' });
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'POST',
+      path: '/_connector/_sync_job',
+      body: {
+        id: 'connectorId',
+        job_type: 'full',
       },
-      index: CURRENT_CONNECTORS_JOB_INDEX,
     });
   });
 
-  it('should not create index if there is no connector', async () => {
-    mockClient.get.mockImplementation(() => {
-      return Promise.resolve({});
-    });
+  it('should start an incremental sync', async () => {
+    mockClient.transport.request.mockImplementation(() => ({ id: '12345' }));
+
     await expect(
       startConnectorSync(mockClient as unknown as ElasticsearchClient, {
         connectorId: 'connectorId',
-        jobType: SyncJobType.FULL,
+        jobType: SyncJobType.INCREMENTAL,
       })
-    ).rejects.toEqual(new Error('resource_not_found'));
-    expect(mockClient.index).not.toHaveBeenCalled();
+    ).resolves.toEqual({ id: '12345' });
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'POST',
+      path: '/_connector/_sync_job',
+      body: {
+        id: 'connectorId',
+        job_type: 'incremental',
+      },
+    });
   });
 
-  it('should start an incremental sync', async () => {
-    mockClient.get.mockImplementation(() => {
-      return Promise.resolve({
-        _id: 'connectorId',
-        _source: {
-          api_key_id: null,
-          configuration: {},
-          created_at: null,
-          custom_scheduling: {},
-          error: null,
-          filtering: [],
-          index_name: 'index_name',
-          language: null,
-          last_access_control_sync_status: null,
-          last_seen: null,
-          last_sync_error: null,
-          last_sync_scheduled_at: null,
-          last_sync_status: null,
-          last_synced: null,
-          scheduling: { enabled: true, interval: '1 2 3 4 5' },
-          service_type: null,
-          status: 'not connected',
-          sync_now: false,
-        },
-        index: CONNECTORS_INDEX,
-      });
-    });
-    mockClient.index.mockImplementation(() => ({ _id: 'fakeId' }));
+  it('should start an access_control sync', async () => {
+    mockClient.transport.request.mockImplementation(() => ({ id: '12345' }));
 
     await expect(
       startConnectorSync(mockClient as unknown as ElasticsearchClient, {
         connectorId: 'connectorId',
-        jobType: SyncJobType.INCREMENTAL,
+        jobType: SyncJobType.ACCESS_CONTROL,
       })
-    ).resolves.toEqual({ _id: 'fakeId' });
-    expect(mockClient.index).toHaveBeenCalledWith({
-      document: {
-        cancelation_requested_at: null,
-        canceled_at: null,
-        completed_at: null,
-        connector: {
-          configuration: {},
-          filtering: null,
-          id: 'connectorId',
-          index_name: 'index_name',
-          language: null,
-          pipeline: null,
-          service_type: null,
-        },
-        created_at: expect.any(String),
-        deleted_document_count: 0,
-        error: null,
-        indexed_document_count: 0,
-        indexed_document_volume: 0,
-        job_type: SyncJobType.INCREMENTAL,
-        last_seen: null,
-        metadata: {},
-        started_at: null,
-        status: SyncStatus.PENDING,
-        total_document_count: null,
-        trigger_method: TriggerMethod.ON_DEMAND,
-        worker_hostname: null,
+    ).resolves.toEqual({ id: '12345' });
+    expect(mockClient.transport.request).toHaveBeenCalledWith({
+      method: 'POST',
+      path: '/_connector/_sync_job',
+      body: {
+        id: 'connectorId',
+        job_type: 'access_control',
       },
-      index: CURRENT_CONNECTORS_JOB_INDEX,
     });
   });
 
-  it('should start an access control sync', async () => {
-    mockClient.get.mockImplementation(() => {
-      return Promise.resolve({
-        _id: 'connectorId',
-        _source: {
-          api_key_id: null,
-          configuration: {},
-          created_at: null,
-          custom_scheduling: {},
-          error: null,
-          index_name: 'search-index_name',
-          language: null,
-          last_access_control_sync_status: null,
-          last_seen: null,
-          last_sync_error: null,
-          last_sync_scheduled_at: null,
-          last_sync_status: null,
-          last_synced: null,
-          scheduling: { enabled: true, interval: '1 2 3 4 5' },
-          service_type: null,
-          status: 'not connected',
-          sync_now: false,
+  it('sync not started if there is no connector', async () => {
+    const notFoundError = new errors.ResponseError({
+      statusCode: 404,
+      body: {
+        error: {
+          type: `document_missing_exception`,
         },
-        index: CONNECTORS_INDEX,
-      });
-    });
-    mockClient.index.mockImplementation(() => ({ _id: 'fakeId' }));
+      },
+    } as any);
+
+    mockClient.transport.request.mockRejectedValueOnce(notFoundError);
 
     await expect(
       startConnectorSync(mockClient as unknown as ElasticsearchClient, {
         connectorId: 'connectorId',
-        targetIndexName: '.search-acl-filter-index_name',
-        jobType: SyncJobType.ACCESS_CONTROL,
+        jobType: SyncJobType.FULL,
       })
-    ).resolves.toEqual({ _id: 'fakeId' });
-    expect(mockClient.index).toHaveBeenCalledWith({
-      document: {
-        cancelation_requested_at: null,
-        canceled_at: null,
-        completed_at: null,
-        connector: {
-          configuration: {},
-          filtering: null,
-          id: 'connectorId',
-          index_name: `${CONNECTORS_ACCESS_CONTROL_INDEX_PREFIX}index_name`,
-          language: null,
-          pipeline: null,
-          service_type: null,
-        },
-        created_at: expect.any(String),
-        deleted_document_count: 0,
-        error: null,
-        indexed_document_count: 0,
-        indexed_document_volume: 0,
-        job_type: SyncJobType.ACCESS_CONTROL,
-        last_seen: null,
-        metadata: {},
-        started_at: null,
-        status: SyncStatus.PENDING,
-        total_document_count: null,
-        trigger_method: TriggerMethod.ON_DEMAND,
-        worker_hostname: null,
-      },
-      index: CURRENT_CONNECTORS_JOB_INDEX,
-    });
+    ).rejects.toEqual(notFoundError);
   });
 });