feat(events,github-org): add support for EventsService, events at new backend

Adds optional support for the EventsService passed
as argument to the factory method.

For the new backend, the EventsService is used as dependency and passed on.
Events support will be active always.

The support for the deprecated EventBroker and its interfaces was kept for
easier migration and will be removed as a separate follow-up change that
could go into a following release.

Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>

Author: pjungermann

.changeset/happy-walls-think.md

@@ -0,0 +1,33 @@
+---
+'@backstage/plugin-catalog-backend-module-github-org': patch
+'@backstage/plugin-catalog-backend-module-github': patch
+---
+
+Support EventsService and events with the new backend system (through EventsService) for `GithubOrgEntityProvider` and `GithubMultiOrgEntityProvider`.
+
+_New/Current Backend System:_
+
+The events support for the provider will be enabled always, making it ready to consume events from its subscribed topics.
+In order to receive events and make use of this feature, you still need to set up receiving events from the event source as before.
+
+_Legacy Backend System:_
+
+You can pass the `EventsService` instance to the factory method as one of its options:
+
+```diff
+  // packages/backend/src/plugins/catalog.ts
+  const githubOrgProvider = GithubOrgEntityProvider.fromConfig(env.config, {
+    events: env.events,
+    // ...
+  });
+- env.eventBroker.subscribe(githubOrgProvider);
+```
+
+```diff
+  // packages/backend/src/plugins/catalog.ts
+  const githubMultiOrgProvider = GithubMultiOrgEntityProvider.fromConfig(env.config, {
+    events: env.events,
+    // ...
+  });
+- env.eventBroker.subscribe(githubMultiOrgProvider);
+```

docs/integrations/github/org.md

@@ -103,6 +103,32 @@ export default async function createPlugin(
 
 ## Installation with Events Support
 
+_For the legacy backend system, please read the subsection below._
+
+The catalog module `github-org` comes with events support enabled for the `GithubMultiOrgEntityProvider`.
+This will make it subscribe to its relevant topics and expects these events to be published via the `EventsService`.
+
+Topics:
+
+- `github.installation`
+- `github.membership`
+- `github.organization`
+- `github.team`
+
+Additionally, you should install the
+[event router by `events-backend-module-github`](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-github/README.md)
+which will route received events from the generic topic `github` to more specific ones
+based on the event type (e.g., `github.membership`).
+
+In order to receive Webhook events by GitHub, you have to decide how you want them
+to be ingested into Backstage and published to its `EventsService`.
+You can decide between the following options (extensible):
+
+- [via HTTP endpoint](https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md)
+- [via an AWS SQS queue](https://github.com/backstage/backstage/tree/master/plugins/events-backend-module-aws-sqs/README.md)
+
+### Legacy Backend System
+
 Please follow the installation instructions at
 
 - <https://github.com/backstage/backstage/tree/master/plugins/events-backend/README.md>
@@ -133,12 +159,12 @@ export default async function createPlugin(
     id: 'production',
     orgUrl: 'https://github.com/backstage',
     logger: env.logger,
+    events: env.events,
     schedule: env.scheduler.createScheduledTaskRunner({
       frequency: { minutes: 60 },
       timeout: { minutes: 15 },
     }),
   });
-  env.eventBroker.subscribe(githubOrgProvider);
   builder.addEntityProvider(githubOrgProvider);
   /* highlight-add-end */
   const { processingEngine, router } = await builder.build();
@@ -169,12 +195,11 @@ export default async function createPlugin(
       // also omit this option to ingest all orgs accessible by your GitHub integration
       orgs: ['org-a', 'org-b'],
       logger: env.logger,
+      events: env.events,
       schedule: env.scheduler.createScheduledTaskRunner({
         frequency: { minutes: 60 },
         timeout: { minutes: 15 },
       }),
-      // Pass in the eventBroker to allow this provider to subscribe to GitHub events
-      eventBroker: env.eventBroker,
     }),
   );
   /* highlight-add-end */

plugins/catalog-backend-module-github-org/package.json

@@ -36,7 +36,8 @@
     "@backstage/backend-tasks": "workspace:^",
     "@backstage/config": "workspace:^",
     "@backstage/plugin-catalog-backend-module-github": "workspace:^",
-    "@backstage/plugin-catalog-node": "workspace:^"
+    "@backstage/plugin-catalog-node": "workspace:^",
+    "@backstage/plugin-events-node": "workspace:^"
   },
   "devDependencies": {
     "@backstage/backend-test-utils": "workspace:^",

plugins/catalog-backend-module-github-org/src/module.ts

@@ -31,6 +31,7 @@ import {
   UserTransformer,
 } from '@backstage/plugin-catalog-backend-module-github';
 import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha';
+import { eventsServiceRef } from '@backstage/plugin-events-node';
 
 /**
  * Interface for {@link githubOrgEntityProviderTransformsExtensionPoint}.
@@ -95,16 +96,18 @@ export const catalogModuleGithubOrgEntityProvider = createBackendModule({
       deps: {
         catalog: catalogProcessingExtensionPoint,
         config: coreServices.rootConfig,
+        events: eventsServiceRef,
         logger: coreServices.logger,
         scheduler: coreServices.scheduler,
       },
-      async init({ catalog, config, logger, scheduler }) {
+      async init({ catalog, config, events, logger, scheduler }) {
         for (const definition of readDefinitionsFromConfig(config)) {
           catalog.addEntityProvider(
             GithubMultiOrgEntityProvider.fromConfig(config, {
               id: definition.id,
               githubUrl: definition.githubUrl,
               orgs: definition.orgs,
+              events,
               schedule: scheduler.createScheduledTaskRunner(
                 definition.schedule,
               ),

plugins/catalog-backend-module-github/api-report.md

@@ -142,6 +142,7 @@ export type GithubMultiOrgConfig = Array<{
 // @public
 export class GithubMultiOrgEntityProvider implements EntityProvider {
   constructor(options: {
+    events?: EventsService;
     id: string;
     gitHubConfig: GithubIntegrationConfig;
     githubCredentialsProvider: GithubCredentialsProvider;
@@ -165,7 +166,9 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
 
 // @public
 export interface GithubMultiOrgEntityProviderOptions {
+  // @deprecated
   eventBroker?: EventBroker;
+  events?: EventsService;
   githubCredentialsProvider?: GithubCredentialsProvider;
   githubUrl: string;
   id: string;
@@ -220,6 +223,7 @@ export class GithubOrgEntityProvider
   implements EntityProvider, EventSubscriber
 {
   constructor(options: {
+    events?: EventsService;
     id: string;
     orgUrl: string;
     gitHubConfig: GithubIntegrationConfig;
@@ -249,6 +253,7 @@ export type GitHubOrgEntityProviderOptions = GithubOrgEntityProviderOptions;
 
 // @public
 export interface GithubOrgEntityProviderOptions {
+  events?: EventsService;
   githubCredentialsProvider?: GithubCredentialsProvider;
   id: string;
   logger: Logger;

plugins/catalog-backend-module-github/src/providers/GithubMultiOrgEntityProvider.ts

@@ -38,7 +38,11 @@ import {
   EntityProvider,
   EntityProviderConnection,
 } from '@backstage/plugin-catalog-node';
-import { EventBroker, EventParams } from '@backstage/plugin-events-node';
+import {
+  EventBroker,
+  EventParams,
+  EventsService,
+} from '@backstage/plugin-events-node';
 import { graphql } from '@octokit/graphql';
 import {
   InstallationCreatedEvent,
@@ -80,6 +84,13 @@ import {
 import { splitTeamSlug } from '../lib/util';
 import { areGroupEntities, areUserEntities } from '../lib/guards';
 
+const EVENT_TOPICS = [
+  'github.installation',
+  'github.membership',
+  'github.organization',
+  'github.team',
+];
+
 /**
  * Options for {@link GithubMultiOrgEntityProvider}.
  *
@@ -101,11 +112,16 @@ export interface GithubMultiOrgEntityProviderOptions {
   githubUrl: string;
 
   /**
-   * The list of the GitHub orgs to consume. By default will consume all accessible
+   * The list of the GitHub orgs to consume. By default, it will consume all accessible
    * orgs on the given GitHub instance (support for GitHub App integration only).
    */
   orgs?: string[];
 
+  /**
+   * Passing the optional EventsService enables event-based delta updates.
+   */
+  events?: EventsService;
+
   /**
    * The refresh schedule to use.
    *
@@ -138,12 +154,14 @@ export interface GithubMultiOrgEntityProviderOptions {
 
   /**
    * Optionally include a team transformer for transforming from GitHub teams to Group Entities.
-   * By default groups will be namespaced according to their GitHub org.
+   * By default, groups will be namespaced according to their GitHub org.
    */
   teamTransformer?: TeamTransformer;
 
   /**
    * An EventBroker to subscribe this provider to GitHub events to trigger delta mutations
+   *
+   * @deprecated Use `events` instead.
    */
   eventBroker?: EventBroker;
 }
@@ -206,6 +224,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
 
   constructor(
     private readonly options: {
+      events?: EventsService;
       id: string;
       gitHubConfig: GithubIntegrationConfig;
       githubCredentialsProvider: GithubCredentialsProvider;
@@ -225,6 +244,11 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
   /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
   async connect(connection: EntityProviderConnection) {
     this.connection = connection;
+    await this.options.events?.subscribe({
+      id: this.getProviderName(),
+      topics: EVENT_TOPICS,
+      onEvent: params => this.onEvent(params),
+    });
     await this.scheduleFn?.();
   }
 
@@ -312,12 +336,7 @@ export class GithubMultiOrgEntityProvider implements EntityProvider {
   }
 
   private supportsEventTopics(): string[] {
-    return [
-      'github.installation',
-      'github.organization',
-      'github.team',
-      'github.membership',
-    ];
+    return EVENT_TOPICS;
   }
 
   private async onEvent(params: EventParams): Promise<void> {

plugins/catalog-backend-module-github/src/providers/GithubOrgEntityProvider.ts

@@ -28,7 +28,11 @@ import {
   EntityProvider,
   EntityProviderConnection,
 } from '@backstage/plugin-catalog-node';
-import { EventParams, EventSubscriber } from '@backstage/plugin-events-node';
+import {
+  EventParams,
+  EventsService,
+  EventSubscriber,
+} from '@backstage/plugin-events-node';
 import { graphql } from '@octokit/graphql';
 import {
   MembershipEvent,
@@ -62,6 +66,12 @@ import { parseGithubOrgUrl } from '../lib/util';
 import { withLocations } from '../lib/withLocations';
 import { areGroupEntities, areUserEntities } from '../lib/guards';
 
+const EVENT_TOPICS = [
+  'github.membership',
+  'github.organization',
+  'github.team',
+];
+
 /**
  * Options for {@link GithubOrgEntityProvider}.
  *
@@ -82,6 +92,11 @@ export interface GithubOrgEntityProviderOptions {
    */
   orgUrl: string;
 
+  /**
+   * Passing the optional EventsService enables event-based delta updates.
+   */
+  events?: EventsService;
+
   /**
    * The refresh schedule to use.
    *
@@ -163,6 +178,7 @@ export class GithubOrgEntityProvider
 
   constructor(
     private options: {
+      events?: EventsService;
       id: string;
       orgUrl: string;
       gitHubConfig: GithubIntegrationConfig;
@@ -185,6 +201,11 @@ export class GithubOrgEntityProvider
   /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
   async connect(connection: EntityProviderConnection) {
     this.connection = connection;
+    await this.options.events?.subscribe({
+      id: this.getProviderName(),
+      topics: EVENT_TOPICS,
+      onEvent: params => this.onEvent(params),
+    });
     await this.scheduleFn?.();
   }
 
@@ -315,7 +336,7 @@ export class GithubOrgEntityProvider
 
   /** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
   supportsEventTopics(): string[] {
-    return ['github.organization', 'github.team', 'github.membership'];
+    return EVENT_TOPICS;
   }
 
   private async onTeamEditedInOrganization(

yarn.lock

@@ -5593,6 +5593,7 @@ __metadata:
     "@backstage/config": "workspace:^"
     "@backstage/plugin-catalog-backend-module-github": "workspace:^"
     "@backstage/plugin-catalog-node": "workspace:^"
+    "@backstage/plugin-events-node": "workspace:^"
     luxon: ^3.0.0
   languageName: unknown
   linkType: soft