From 8fe67b20468454a09964b8ce0540f77d01ebff9e Mon Sep 17 00:00:00 2001 From: Tigran Mkrtchyan Date: Mon, 7 Aug 2023 13:53:52 +0200 Subject: [PATCH] chimera: introduce attribute to control attribute update consistency Motivation: with 9.0 chimera have introduced to java properties to control behaviour of parent directory attribute update policy. As java properties are not practical for admin use a regular dcache property is required. Modification: Introduce `chimera.attr-consistency` property that accepts "strong", "weak" and "soft" values to control the behaviour. Updates JdbcFs constructor to pass the value through the code. Updated nfs door, pnfs manager, resilience and qos xml files to pass the value. Update docu. Result: admin friendly way to control attribute consistency behaviour. Acked-by: Albert Rossi Target: master Require-book: yes Require-notes: yes --- .../src/main/markdown/config-chimera.md | 26 ++++++++++++++++ .../org/dcache/chimera/CreateBenchmark.java | 16 ++-------- .../java/org/dcache/chimera/FsFactory.java | 2 +- .../java/org/dcache/chimera/FsSqlDriver.java | 5 +-- .../main/java/org/dcache/chimera/JdbcFs.java | 14 ++++++--- .../dcache/chimera/PgSQL95FsSqlDriver.java | 19 ++++++++++-- .../dcache/chimera/spi/DBDriverProvider.java | 3 +- .../chimera/spi/H2DBDriverProvider.java | 2 +- .../chimera/spi/HsqlDBDriverProvider.java | 2 +- .../chimera/spi/PgSQLDrivertProvider.java | 4 +-- .../dcache/chimera/ChimeraTestCaseHelper.java | 2 +- .../org/dcache/chimera/DCacheAwareJdbcFs.java | 8 ++--- .../org/dcache/chimera/PerformanceTest.java | 2 +- .../namespace/pnfsmanager-chimera.xml | 1 + .../chimera/nfsv41/door/nfsv41-common.xml | 1 + .../resources/org/dcache/qos/qos-scanner.xml | 1 + .../org/dcache/resilience/resilience.xml | 1 + skel/share/defaults/chimera.properties | 31 +++++++++++++++++++ 18 files changed, 105 insertions(+), 35 deletions(-) diff --git a/docs/TheBook/src/main/markdown/config-chimera.md b/docs/TheBook/src/main/markdown/config-chimera.md index 65b9904407f..5f482901fa8 100644 --- a/docs/TheBook/src/main/markdown/config-chimera.md +++ b/docs/TheBook/src/main/markdown/config-chimera.md @@ -48,6 +48,32 @@ chimera.db.user=myuser chimera.db.password=secret ``` +## Attribute consistency policy + +On new filesystem object creation in a directory the `modification` and +`change id` attributes must be updated to provide a consistent, up-to-date view of the changes. +In highly concurrent environments such updates might create so-called `hot inodes` +and serialize all updates in a single directory, thus, reducing the namespace throughput. + +As such strong consistency is not always required, to improve concurrent updates to a single directory +the POSIX constraints can be relaxed. The `chimera.attr-consistency` attribute controls the namespace attribute +update bahaviour of a parent directory on update: + +| policy | behaviour | +|:-------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| strong | a creation of a filesystem object will right away update parent directory's mtime, ctime, nlink and generation attributes | +| weak | a creation of a filesystem object will eventually update (after 30 seconds) parent directory's mtime, ctime, nlink and generation attributes. Multiple concurrent modifications to a directory are aggregated into single attribute update. | +| soft | same as weak, however, reading of directory attributes will take into account pending attribute updates. | + +Read-write exported NFS doors SHOULD run with `strong consistency` or `soft consistency` to maintain POSIX +compliance. Read-only NFS doors might run with `weak consistency` if non-up-to-date directory attributes can +be tolerated, for example when accessing existing data, or `soft consistency`, if up-to-date information +is desired, typically when seeking for newly arrived files through other doors. + +``` +chimera.attr-consistency=strong +``` + ## Mounting Chimera through NFS dCache does not need the Chimera filesystem to be mounted, but a mounted file system is convenient for administrative access. This offers the opportunity to use OS-level tools like `ls` and `mkdir` for Chimera. However, direct I/O-operations like `cp` are not possible, since the `NFSv3` interface provides the namespace part only. This section describes how to start the Chimera `NFSv3` server and mount the name space. diff --git a/modules/benchmarks/src/main/java/org/dcache/chimera/CreateBenchmark.java b/modules/benchmarks/src/main/java/org/dcache/chimera/CreateBenchmark.java index 56e644d1edf..01eefdad844 100644 --- a/modules/benchmarks/src/main/java/org/dcache/chimera/CreateBenchmark.java +++ b/modules/benchmarks/src/main/java/org/dcache/chimera/CreateBenchmark.java @@ -52,7 +52,7 @@ public class CreateBenchmark { @State(Scope.Benchmark) public static class DB { - @Param(value = {"weak", "strong", "week_softupdate"}) + @Param(value = {"weak", "strong", "soft"}) String wcc; protected HikariDataSource _dataSource; @@ -65,18 +65,6 @@ public static class DB { @Setup public void setUp() throws IOException, SQLException, LiquibaseException { - switch (wcc) { - case "week_softupdate": - System.setProperty("chimera_soft_update", "true"); - case "weak": - System.setProperty("chimera_lazy_wcc", "true"); - case "strong": - ; - break; - default: - throw new IllegalArgumentException("Invalid wcc mode: " + wcc); - } - Properties dbProperties = new Properties(); try (InputStream input = Resources.asByteSource(DB_TEST_PROPERTIES).openStream()) { dbProperties.load(input); @@ -111,7 +99,7 @@ public void setUp() throws IOException, SQLException, LiquibaseException { } PlatformTransactionManager txManager = new DataSourceTransactionManager(_dataSource); - _fs = new JdbcFs(_dataSource, txManager); + _fs = new JdbcFs(_dataSource, txManager, wcc); _rootInode = _fs.path2inode("/"); _fs.createTag(_rootInode, "aTag"); FsInode tagInode = new FsInode_TAG(_fs, _rootInode.ino(), "aTag"); diff --git a/modules/chimera/src/main/java/org/dcache/chimera/FsFactory.java b/modules/chimera/src/main/java/org/dcache/chimera/FsFactory.java index e85da7c9d5a..2a5ce794a2b 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/FsFactory.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/FsFactory.java @@ -31,7 +31,7 @@ public static FileSystemProvider createFileSystem(String url, String user, Strin AlarmEnabledDataSource dataSource = new AlarmEnabledDataSource(url, FsFactory.class.getSimpleName(), getDataSource(url, user, password)); PlatformTransactionManager txManager = new DataSourceTransactionManager(dataSource); - return new JdbcFs(dataSource, txManager) { + return new JdbcFs(dataSource, txManager, "strong") { @Override public void close() throws IOException { super.close(); diff --git a/modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java b/modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java index 5a7085aaa9c..8f62fe294f1 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java @@ -1807,14 +1807,15 @@ String resolvePath(FsInode root, String path) throws ChimeraFsException, SQLExce * specific driver not available * * @param dataSource database data source + * @param consistency desired file system attribute consistency * @return FsSqlDriver */ - static FsSqlDriver getDriverInstance(DataSource dataSource) + static FsSqlDriver getDriverInstance(DataSource dataSource, String consistency) throws ChimeraFsException, SQLException { for (DBDriverProvider driverProvider : ALL_PROVIDERS) { if (driverProvider.isSupportDB(dataSource)) { - FsSqlDriver driver = driverProvider.getDriver(dataSource); + FsSqlDriver driver = driverProvider.getDriver(dataSource, consistency); LOGGER.info("Using DBDriverProvider: {}", driver.getClass().getName()); return driver; } diff --git a/modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java b/modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java index e8eb20943a3..e69f1e8f59f 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java @@ -184,6 +184,10 @@ public FsStat load(Object k) throws Exception { */ private RetentionPolicy _defaultRetentionPolicy; + /** + * File attribute consistency policy. + */ + private final String _attributeConsistency; private final ScheduledExecutorService maintenanceTaskExecutor = Executors.newSingleThreadScheduledExecutor( new ThreadFactoryBuilder() @@ -194,20 +198,21 @@ public FsStat load(Object k) throws Exception { private ScheduledFuture maintenanceTask; - public JdbcFs(DataSource dataSource, PlatformTransactionManager txManager) + public JdbcFs(DataSource dataSource, PlatformTransactionManager txManager, String consistency) throws SQLException, ChimeraFsException { - this(dataSource, txManager, 0); + this(dataSource, txManager, 0, consistency); } - public JdbcFs(DataSource dataSource, PlatformTransactionManager txManager, int id) + public JdbcFs(DataSource dataSource, PlatformTransactionManager txManager, int id, String consistency) throws SQLException, ChimeraFsException { _dbConnectionsPool = dataSource; _fsId = id; _tx = txManager; + _attributeConsistency = consistency; // try to get database dialect specific query engine - _sqlDriver = FsSqlDriver.getDriverInstance(dataSource); + _sqlDriver = FsSqlDriver.getDriverInstance(dataSource, _attributeConsistency); } public void setQuota(QuotaHandler quota) { @@ -1482,6 +1487,7 @@ public String getInfo() { sb.append("rootID : ").append(e.getMessage()).append('\n'); } sb.append("FsId : ").append(_fsId).append('\n'); + sb.append("Wcc : ").append(_attributeConsistency).append("\n"); return sb.toString(); } diff --git a/modules/chimera/src/main/java/org/dcache/chimera/PgSQL95FsSqlDriver.java b/modules/chimera/src/main/java/org/dcache/chimera/PgSQL95FsSqlDriver.java index bab3e0fecff..e83848fc23c 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/PgSQL95FsSqlDriver.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/PgSQL95FsSqlDriver.java @@ -63,13 +63,26 @@ public class PgSQL95FsSqlDriver extends FsSqlDriver { /** * this is a utility class which is issues SQL queries on database */ - public PgSQL95FsSqlDriver(DataSource dataSource) throws ChimeraFsException { + public PgSQL95FsSqlDriver(DataSource dataSource, String consistency) throws ChimeraFsException { super(dataSource); LOGGER.info("Running PostgreSQL >= 9.5 specific Driver"); this.dataSource = dataSource; - enableLazyWcc = System.getProperty("chimera_lazy_wcc") != null; - enableSoftUpdate = enableLazyWcc && System.getProperty("chimera_soft_update") != null; + boolean softUpdateTmp = false; + switch (consistency) { + case "soft": + softUpdateTmp = true; + // fallthrough + case "weak": + enableSoftUpdate = softUpdateTmp; + enableLazyWcc = true; + break; + case "strong": + enableLazyWcc = false; + enableSoftUpdate = false; + default: + throw new IllegalArgumentException("Unsupported attribute consistency option '" + consistency + "'"); + } createProcedureName = enableLazyWcc ? "f_create_inode95_lazy_wcc" : "f_create_inode95"; } diff --git a/modules/chimera/src/main/java/org/dcache/chimera/spi/DBDriverProvider.java b/modules/chimera/src/main/java/org/dcache/chimera/spi/DBDriverProvider.java index 63022c40ea3..ba188d5b59e 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/spi/DBDriverProvider.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/spi/DBDriverProvider.java @@ -26,8 +26,9 @@ public interface DBDriverProvider { * Get {@link FsSqlDriver} for the specific database. * * @param dataSource source for database connection + * @param consistency desired file attribute consistency * @return driver for specific database. * @throws SQLException on db errors */ - FsSqlDriver getDriver(DataSource dataSource) throws SQLException, ChimeraFsException; + FsSqlDriver getDriver(DataSource dataSource, String consistency) throws SQLException, ChimeraFsException; } diff --git a/modules/chimera/src/main/java/org/dcache/chimera/spi/H2DBDriverProvider.java b/modules/chimera/src/main/java/org/dcache/chimera/spi/H2DBDriverProvider.java index e41a55a7d39..5910f65f732 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/spi/H2DBDriverProvider.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/spi/H2DBDriverProvider.java @@ -26,7 +26,7 @@ public boolean isSupportDB(DataSource dataSource) throws SQLException { } @Override - public FsSqlDriver getDriver(DataSource dataSource) throws SQLException, ChimeraFsException { + public FsSqlDriver getDriver(DataSource dataSource, String consistency) throws SQLException, ChimeraFsException { return new H2FsSqlDriver(dataSource); } } diff --git a/modules/chimera/src/main/java/org/dcache/chimera/spi/HsqlDBDriverProvider.java b/modules/chimera/src/main/java/org/dcache/chimera/spi/HsqlDBDriverProvider.java index e3b93694138..4f8b6818dd0 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/spi/HsqlDBDriverProvider.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/spi/HsqlDBDriverProvider.java @@ -26,7 +26,7 @@ public boolean isSupportDB(DataSource dataSource) throws SQLException { } @Override - public FsSqlDriver getDriver(DataSource dataSource) throws SQLException, ChimeraFsException { + public FsSqlDriver getDriver(DataSource dataSource, String consistency) throws SQLException, ChimeraFsException { return new HsqlDBFsSqlDriver(dataSource); } diff --git a/modules/chimera/src/main/java/org/dcache/chimera/spi/PgSQLDrivertProvider.java b/modules/chimera/src/main/java/org/dcache/chimera/spi/PgSQLDrivertProvider.java index 2b71fdfd09b..126e8f9111e 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/spi/PgSQLDrivertProvider.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/spi/PgSQLDrivertProvider.java @@ -43,7 +43,7 @@ public boolean isSupportDB(DataSource dataSource) throws SQLException { } @Override - public FsSqlDriver getDriver(DataSource dataSource) throws SQLException, ChimeraFsException { + public FsSqlDriver getDriver(DataSource dataSource, String consistency) throws SQLException, ChimeraFsException { Connection dbConnection = null; try { @@ -53,7 +53,7 @@ public FsSqlDriver getDriver(DataSource dataSource) throws SQLException, Chimera int min = dbConnection.getMetaData().getDatabaseMinorVersion(); if ((maj > 9) || (maj == 9 && min >= 5)) { - return new PgSQL95FsSqlDriver(dataSource); + return new PgSQL95FsSqlDriver(dataSource, consistency); } else { throw new IllegalArgumentException("Required PostgreSQL 9.5 or newer"); } diff --git a/modules/chimera/src/test/java/org/dcache/chimera/ChimeraTestCaseHelper.java b/modules/chimera/src/test/java/org/dcache/chimera/ChimeraTestCaseHelper.java index 5d5c8acddb6..1d1f975dc30 100644 --- a/modules/chimera/src/test/java/org/dcache/chimera/ChimeraTestCaseHelper.java +++ b/modules/chimera/src/test/java/org/dcache/chimera/ChimeraTestCaseHelper.java @@ -50,7 +50,7 @@ public void setUp() throws Exception { } PlatformTransactionManager txManager = new DataSourceTransactionManager(_dataSource); - _fs = new JdbcFs(_dataSource, txManager); + _fs = new JdbcFs(_dataSource, txManager, "strong"); _rootInode = _fs.path2inode("/"); } diff --git a/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java b/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java index 7a9c5e8b2fc..1264ac57e9c 100644 --- a/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java +++ b/modules/dcache-chimera/src/main/java/org/dcache/chimera/DCacheAwareJdbcFs.java @@ -182,14 +182,14 @@ public void setQueryPnfsManagerOnRename(boolean yes) { queryPnfsManagerOnRename = yes; } - public DCacheAwareJdbcFs(DataSource dataSource, PlatformTransactionManager txManager) + public DCacheAwareJdbcFs(DataSource dataSource, PlatformTransactionManager txManager, String consistency) throws ChimeraFsException, SQLException { - super(dataSource, txManager); + super(dataSource, txManager, consistency); } - public DCacheAwareJdbcFs(DataSource dataSource, PlatformTransactionManager txManager, int id) + public DCacheAwareJdbcFs(DataSource dataSource, PlatformTransactionManager txManager, int id, String consistency) throws ChimeraFsException, SQLException { - super(dataSource, txManager, id); + super(dataSource, txManager, id, consistency); } public void setPnfsHandler(PnfsHandler pnfsHandler) { diff --git a/modules/dcache-chimera/src/main/java/org/dcache/chimera/PerformanceTest.java b/modules/dcache-chimera/src/main/java/org/dcache/chimera/PerformanceTest.java index acbb02ba11a..1c48a0ac787 100644 --- a/modules/dcache-chimera/src/main/java/org/dcache/chimera/PerformanceTest.java +++ b/modules/dcache-chimera/src/main/java/org/dcache/chimera/PerformanceTest.java @@ -193,7 +193,7 @@ public static void main(String arguments[]) throws Exception { System.out.println("Starting chimera... "); HikariDataSource dataSource = FsFactory.getDataSource(jdbc, user, password); PlatformTransactionManager txManager = new DataSourceTransactionManager(dataSource); - FileSystemProvider fileSystem = new JdbcFs(dataSource, txManager); + FileSystemProvider fileSystem = new JdbcFs(dataSource, txManager, "strong"); provider = new ChimeraNameSpaceProvider(); provider.setAclEnabled(false); provider.setExtractor(new ChimeraOsmStorageInfoExtractor(StorageInfo.DEFAULT_ACCESS_LATENCY, diff --git a/modules/dcache-chimera/src/main/resources/diskCacheV111/namespace/pnfsmanager-chimera.xml b/modules/dcache-chimera/src/main/resources/diskCacheV111/namespace/pnfsmanager-chimera.xml index 8bd1c78d279..75feb3b847a 100644 --- a/modules/dcache-chimera/src/main/resources/diskCacheV111/namespace/pnfsmanager-chimera.xml +++ b/modules/dcache-chimera/src/main/resources/diskCacheV111/namespace/pnfsmanager-chimera.xml @@ -102,6 +102,7 @@ Chimera + diff --git a/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml b/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml index 7e3f5a198a0..cd8c01f7498 100644 --- a/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml +++ b/modules/dcache-nfs/src/main/resources/org/dcache/chimera/nfsv41/door/nfsv41-common.xml @@ -134,6 +134,7 @@ Chimera Filesystem + diff --git a/modules/dcache-qos/src/main/resources/org/dcache/qos/qos-scanner.xml b/modules/dcache-qos/src/main/resources/org/dcache/qos/qos-scanner.xml index 7fe0daff03b..19a49c2b217 100644 --- a/modules/dcache-qos/src/main/resources/org/dcache/qos/qos-scanner.xml +++ b/modules/dcache-qos/src/main/resources/org/dcache/qos/qos-scanner.xml @@ -104,6 +104,7 @@ File system implementation (Chimera). + diff --git a/modules/dcache-resilience/src/main/resources/org/dcache/resilience/resilience.xml b/modules/dcache-resilience/src/main/resources/org/dcache/resilience/resilience.xml index de42e7d8fb5..8da0f8ece2a 100644 --- a/modules/dcache-resilience/src/main/resources/org/dcache/resilience/resilience.xml +++ b/modules/dcache-resilience/src/main/resources/org/dcache/resilience/resilience.xml @@ -60,6 +60,7 @@ Chimera + diff --git a/skel/share/defaults/chimera.properties b/skel/share/defaults/chimera.properties index fca0497478d..bc392ede860 100644 --- a/skel/share/defaults/chimera.properties +++ b/skel/share/defaults/chimera.properties @@ -31,6 +31,37 @@ chimera.db.password.file = ${dcache.db.password.file} chimera.db.schema.changelog = org/dcache/chimera/changelog/changelog-master.xml + +# ---- File attribute consistency behaviour +# +# On new filesystem object creation in a directory the `modification` and +# `change id` must be updated to provide a consistent, up-to-date view of the changes. +# In highly concurrent environments such updates might create so-called 'hot inodes' +# and serialize all updates in a single directory, thus, reducing the namespace throughput. +# +# As such strong consistency is not always required, to improve concurrent updates to +# a single directory this requirement can be relaxed. +# +# The following values control parent directory attribute consistency: +# +# strong: a creation of a filesystem object will right away update parent directory's +# mtime, ctime, nlink and generation attributes +# +# weak : a creation of a filesystem object will eventually update (after 30 seconds) parent +# directory's mtime, ctime, nlink and generation attributes. Multiple concurrent +# modifications to a directory are aggregated into single attribute update. +# +# soft : same as weak, however, reading of directory attributes will take into account +# pending attribute updates. +# +# +# Read-write exported NFS doors SHOULD run with `strong consistency` or `soft consistency` to maintain POSIX +# compliance. Read-only NFS doors might run with `weak consistency` if non-up-to-date directory attributes can +# be tolerated, for example when accessing existing data, or `soft consistency`, if up-to-date information +# is desired, typically when seeking for newly arrived files through other doors. +# +(one-of?strong|weak|soft)chimera.attr-consistency=strong + (obsolete)chimera.db.dialect = Not used any more (obsolete)chimera.db.jar = dCache auto-detects which driver to use (obsolete)chimera.db.jar-when-H2 = dCache auto-detects which driver to use