Skip to content

Commit

Permalink
It turns out that the config handling is quite involved. While trying…
Browse files Browse the repository at this point in the history 
… to move toward #17466 it became clear that the entire config reading layer needed to be concentrated in one place. So this is what this pull request does! Nothing functionally changed; it just starts out by moving all of the config parsing in DatabaseManager out to the config folder. I'll continue on this journey, but the PR would be much too horribly large if this wasn't split into parts :)

Signed-off-by: Fredrik Adelöw <freben@gmail.com>
  • Loading branch information
@freben
freben committed Nov 10, 2023
1 parent 59d1362 commit 6ed76e5
Show file tree
Hide file tree
Showing 15 changed files with 495 additions and 192 deletions.
5 changes: 5 additions & 0 deletions .changeset/poor-buttons-share.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@backstage/backend-common': patch
---

Internal refactor for config
1 change: 1 addition & 0 deletions packages/backend-common/src/database/DatabaseManager.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import { ConfigReader } from '@backstage/config';
import { omit } from 'lodash';
import path from 'path';
Expand Down
214 changes: 23 additions & 191 deletions packages/backend-common/src/database/DatabaseManager.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,34 +14,30 @@
* limitations under the License.
*/

import {
LifecycleService,
LoggerService,
PluginMetadataService,
} from '@backstage/backend-plugin-api';
import { Config, ConfigReader } from '@backstage/config';
import { stringifyError } from '@backstage/errors';
import { JsonObject } from '@backstage/types';
import { Knex } from 'knex';
import { merge, omit } from 'lodash';
import path from 'path';
import { mergeDatabaseConfig } from './config';
import { getClientType } from './config/getClientType';
import { getConfigForPlugin } from './config/getConfigForPlugin';
import { getConnectionConfig } from './config/getConnectionConfig';
import { getDatabaseOverrides } from './config/getDatabaseOverrides';
import { getEnsureExistsConfig } from './config/getEnsureExistsConfig';
import { getPluginDivisionModeConfig } from './config/getPluginDivisionModeConfig';
import { getSchemaOverrides } from './config/getSchemaOverrides';
import {
createDatabaseClient,
createNameOverride,
createSchemaOverride,
ensureDatabaseExists,
ensureSchemaExists,
normalizeConnection,
} from './connection';
import { PluginDatabaseManager } from './types';
import path from 'path';
import {
LifecycleService,
LoggerService,
PluginMetadataService,
} from '@backstage/backend-plugin-api';
import { stringifyError } from '@backstage/errors';

/**
* Provides a config lookup path for a plugin's config block.
*/
function pluginPath(pluginId: string): string {
return `plugin.${pluginId}`;
}

/**
* Creation options for {@link DatabaseManager}.
Expand Down Expand Up @@ -124,9 +120,9 @@ export class DatabaseManager {
* @returns String representing the plugin's database name
*/
private getDatabaseName(pluginId: string): string | undefined {
const connection = this.getConnectionConfig(pluginId);
const connection = getConnectionConfig(this.config, pluginId);

if (this.getClientType(pluginId).client.includes('sqlite3')) {
if (getClientType(this.config, pluginId).client.includes('sqlite3')) {
const sqliteFilename: string | undefined = (
connection as Knex.Sqlite3ConnectionConfig
).filename;
Expand All @@ -144,178 +140,14 @@ export class DatabaseManager {
const databaseName = (connection as Knex.ConnectionConfig)?.database;

// `pluginDivisionMode` as `schema` should use overridden databaseName if supplied or fallback to default knex database
if (this.getPluginDivisionModeConfig() === 'schema') {
if (getPluginDivisionModeConfig(this.config) === 'schema') {
return databaseName;
}

// all other supported databases should fallback to an auto-prefixed name
return databaseName ?? `${this.prefix}${pluginId}`;
}

/**
* Provides the client type which should be used for a given plugin.
*
* The client type is determined by plugin specific config if present.
* Otherwise the base client is used as the fallback.
*
* @param pluginId - Plugin to get the client type for
* @returns Object with client type returned as `client` and boolean
* representing whether or not the client was overridden as
* `overridden`
*/
private getClientType(pluginId: string): {
client: string;
overridden: boolean;
} {
const pluginClient = this.config.getOptionalString(
`${pluginPath(pluginId)}.client`,
);

const baseClient = this.config.getString('client');
const client = pluginClient ?? baseClient;
return {
client,
overridden: client !== baseClient,
};
}

private getRoleConfig(pluginId: string): string | undefined {
return (
this.config.getOptionalString(`${pluginPath(pluginId)}.role`) ??
this.config.getOptionalString('role')
);
}

/**
* Provides the knexConfig which should be used for a given plugin.
*
* @param pluginId - Plugin to get the knexConfig for
* @returns The merged knexConfig value or undefined if it isn't specified
*/
private getAdditionalKnexConfig(pluginId: string): JsonObject | undefined {
const pluginConfig = this.config
.getOptionalConfig(`${pluginPath(pluginId)}.knexConfig`)
?.get<JsonObject>();

const baseConfig = this.config
.getOptionalConfig('knexConfig')
?.get<JsonObject>();

return merge(baseConfig, pluginConfig);
}

private getEnsureExistsConfig(pluginId: string): boolean {
const baseConfig = this.config.getOptionalBoolean('ensureExists') ?? true;
return (
this.config.getOptionalBoolean(`${pluginPath(pluginId)}.ensureExists`) ??
baseConfig
);
}

private getPluginDivisionModeConfig(): string {
return this.config.getOptionalString('pluginDivisionMode') ?? 'database';
}

/**
* Provides a Knex connection plugin config by combining base and plugin
* config.
*
* This method provides a baseConfig for a plugin database connector. If the
* client type has not been overridden, the global connection config will be
* included with plugin specific config as the base. Values from the plugin
* connection take precedence over the base. Base database name is omitted for
* all supported databases excluding SQLite unless `pluginDivisionMode` is set
* to `schema`.
*/
private getConnectionConfig(pluginId: string): Knex.StaticConnectionConfig {
const { client, overridden } = this.getClientType(pluginId);

let baseConnection = normalizeConnection(
this.config.get('connection'),
this.config.getString('client'),
);

if (
client.includes('sqlite3') &&
'filename' in baseConnection &&
baseConnection.filename !== ':memory:'
) {
throw new Error(
'`connection.filename` is not supported for the base sqlite connection. Prefer `connection.directory` or provide a filename for the plugin connection instead.',
);
}

// Databases cannot be shared unless the `pluginDivisionMode` is set to `schema`. The
// `database` property from the base connection is omitted unless `pluginDivisionMode`
// is set to `schema`. SQLite3's `filename` property is an exception as this is used as a
// directory elsewhere so we preserve `filename`.
if (this.getPluginDivisionModeConfig() !== 'schema') {
baseConnection = omit(baseConnection, 'database');
}

// get and normalize optional plugin specific database connection
const connection = normalizeConnection(
this.config.getOptional(`${pluginPath(pluginId)}.connection`),
client,
);

if (client === 'pg') {
(
baseConnection as Knex.PgConnectionConfig
).application_name ||= `backstage_plugin_${pluginId}`;
}

return {
// include base connection if client type has not been overridden
...(overridden ? {} : baseConnection),
...connection,
} as Knex.StaticConnectionConfig;
}

/**
* Provides a Knex database config for a given plugin.
*
* This method provides a Knex configuration object along with the plugin's
* client type.
*
* @param pluginId - The plugin that the database config should correspond with
*/
private getConfigForPlugin(pluginId: string): Knex.Config {
const { client } = this.getClientType(pluginId);
const role = this.getRoleConfig(pluginId);

return {
...this.getAdditionalKnexConfig(pluginId),
client,
connection: this.getConnectionConfig(pluginId),
...(role && { role }),
};
}

/**
* Provides a partial `Knex.Config` database schema override for a given
* plugin.
*
* @param pluginId - Target plugin to get database schema override
* @returns Partial `Knex.Config` with database schema override
*/
private getSchemaOverrides(pluginId: string): Knex.Config | undefined {
return createSchemaOverride(this.getClientType(pluginId).client, pluginId);
}

/**
* Provides a partial `Knex.Config`• database name override for a given plugin.
*
* @param pluginId - Target plugin to get database name override
* @returns Partial `Knex.Config` with database name override
*/
private getDatabaseOverrides(pluginId: string): Knex.Config {
const databaseName = this.getDatabaseName(pluginId);
return databaseName
? createNameOverride(this.getClientType(pluginId).client, databaseName)
: {};
}

/**
* Provides a scoped Knex client for a plugin as per application config.
*
Expand All @@ -336,11 +168,11 @@ export class DatabaseManager {

const clientPromise = Promise.resolve().then(async () => {
const pluginConfig = new ConfigReader(
this.getConfigForPlugin(pluginId) as JsonObject,
getConfigForPlugin(this.config, pluginId) as JsonObject,
);

const databaseName = this.getDatabaseName(pluginId);
if (databaseName && this.getEnsureExistsConfig(pluginId)) {
if (databaseName && getEnsureExistsConfig(this.config, pluginId)) {
try {
await ensureDatabaseExists(pluginConfig, databaseName);
} catch (error) {
Expand All @@ -351,9 +183,9 @@ export class DatabaseManager {
}

let schemaOverrides;
if (this.getPluginDivisionModeConfig() === 'schema') {
schemaOverrides = this.getSchemaOverrides(pluginId);
if (this.getEnsureExistsConfig(pluginId)) {
if (getPluginDivisionModeConfig(this.config) === 'schema') {
schemaOverrides = getSchemaOverrides(this.config, pluginId);
if (getEnsureExistsConfig(this.config, pluginId)) {
try {
await ensureSchemaExists(pluginConfig, pluginId);
} catch (error) {
Expand All @@ -366,7 +198,7 @@ export class DatabaseManager {

const databaseClientOverrides = mergeDatabaseConfig(
{},
this.getDatabaseOverrides(pluginId),
getDatabaseOverrides(this.config, pluginId),
schemaOverrides,
);

Expand Down
41 changes: 41 additions & 0 deletions packages/backend-common/src/database/config/getAdditionalKnexConfig.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import { Config } from '@backstage/config';
import { Knex } from 'knex';
import { merge } from 'lodash';

/**
* Provides the knexConfig which should be used for a given plugin.
*
* @param config - The database config root
* @param pluginId - Plugin to get the knexConfig for
* @returns The merged knexConfig value or undefined if it isn't specified
*/
export function getAdditionalKnexConfig(
config: Config,
pluginId: string,
): Partial<Knex.Config> | undefined {
const pluginConfig = config
.getOptionalConfig(`plugin.${pluginId}.knexConfig`)
?.get<Partial<Knex.Config>>();

const baseConfig = config
.getOptionalConfig('knexConfig')
?.get<Partial<Knex.Config>>();

return merge(baseConfig, pluginConfig);
}
45 changes: 45 additions & 0 deletions packages/backend-common/src/database/config/getClientType.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import { Config } from '@backstage/config';

/**
* Provides the client type which should be used for a given plugin.
*
* The client type is determined by plugin specific config if present.
* Otherwise the base client is used as the fallback.
*
* @param config - The database config root
* @param pluginId - Plugin to get the client type for
* @returns Object with client type returned as `client` and boolean
* representing whether or not the client was overridden as
* `overridden`
*/
export function getClientType(
config: Config,
pluginId: string,
): {
client: string;
overridden: boolean;
} {
const pluginClient = config.getOptionalString(`plugin.${pluginId}.client`);
const baseClient = config.getString('client');
const client = pluginClient ?? baseClient;
return {
client,
overridden: client !== baseClient,
};
}

0 comments on commit 6ed76e5

Please sign in to comment.