From b006e05aec3877a3bf20ab0901d56f3d4a207cea Mon Sep 17 00:00:00 2001 From: alrossi Date: Thu, 16 Jan 2014 08:34:54 -0600 Subject: [PATCH] chimera: add ".(get)(filename)(checksum)" command Adds command ".(get)(filename)(checksum)" OR ".(get)(filename)(checksums)" which returns a comma-delimited list of type:value pairs for all checksums stored in the database. Note that the FileSystemProvider API was extended with a method to retrieve a set of org.dcache.util.Checksum objects, and the FsSqlDriver similarly was provided with the necessary wrapper call to the database. Testing: Works on both nfs3 and fns4.1. Target: master Request: 2.7 Patch: http://rb.dcache.org/r/6405 Requires-book: yes Requires-notes: yes Depends-on: http://rb.dcache.org/r/6404 Acked-by: Tigran BOOK: RELEASE NOTES: A new NFS dot-command has been added to retrieve all the existing checksums for a given file (see body of description). --- .../dcache/chimera/FileSystemProvider.java | 5 + .../java/org/dcache/chimera/FsInodeType.java | 3 +- .../java/org/dcache/chimera/FsInode_PCRC.java | 105 ++++++++++++++++++ .../java/org/dcache/chimera/FsSqlDriver.java | 31 ++++++ .../main/java/org/dcache/chimera/JdbcFs.java | 73 +++++++++--- .../nfsv41/mover/DummyFileSystemProvider.java | 10 ++ 6 files changed, 209 insertions(+), 18 deletions(-) create mode 100644 modules/chimera/src/main/java/org/dcache/chimera/FsInode_PCRC.java diff --git a/modules/chimera/src/main/java/org/dcache/chimera/FileSystemProvider.java b/modules/chimera/src/main/java/org/dcache/chimera/FileSystemProvider.java index 3508afba1aa..8dc667689e3 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/FileSystemProvider.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/FileSystemProvider.java @@ -18,6 +18,7 @@ import java.io.Closeable; import java.util.List; +import java.util.Set; import diskCacheV111.util.AccessLatency; import diskCacheV111.util.RetentionPolicy; @@ -25,6 +26,7 @@ import org.dcache.acl.ACE; import org.dcache.chimera.posix.Stat; import org.dcache.chimera.store.InodeStorageInformation; +import org.dcache.util.Checksum; public interface FileSystemProvider extends Closeable { @@ -276,6 +278,9 @@ public abstract void removeInodeChecksum(FsInode inode, int type) public abstract String getInodeChecksum(FsInode inode, int type) throws ChimeraFsException; + public abstract Set getInodeChecksums(FsInode inode) + throws ChimeraFsException; + public abstract String getInfo(); /** diff --git a/modules/chimera/src/main/java/org/dcache/chimera/FsInodeType.java b/modules/chimera/src/main/java/org/dcache/chimera/FsInodeType.java index 53276f80184..b201370a668 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/FsInodeType.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/FsInodeType.java @@ -28,7 +28,8 @@ public enum FsInodeType { PGET(7), // the content of the inode is the value of requested attributes PSET(8), // by updating mtime of the inode the the defined attribute value is updated CONST(9), // the content of the inode is a free form information - PLOC(10); // the content of the inode is the value of requested attributes + PLOC(10), // the content of the inode is the value of requested attributes + PCRC(11); // the content of the inode is a name-value list of checksum types and checksums private final int _id; diff --git a/modules/chimera/src/main/java/org/dcache/chimera/FsInode_PCRC.java b/modules/chimera/src/main/java/org/dcache/chimera/FsInode_PCRC.java new file mode 100644 index 00000000000..b5ec3b548fc --- /dev/null +++ b/modules/chimera/src/main/java/org/dcache/chimera/FsInode_PCRC.java @@ -0,0 +1,105 @@ +/* + * This library is free software; you can redistribute it and/or modify + * it under the terms of the GNU Library General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this program (see the file COPYING.LIB for more + * details); if not, write to the Free Software Foundation, Inc., + * 675 Mass Ave, Cambridge, MA 02139, USA. + */ +package org.dcache.chimera; + +import java.util.Iterator; +import java.util.Set; + +import org.dcache.chimera.posix.Stat; +import org.dcache.util.Checksum; + +/** + * This class retrieves all the stored checksums for this file, returned as a + * type:value comma-delimited list. + * + * @author arossi + */ +public class FsInode_PCRC extends FsInode { + private String _checksum; + + public FsInode_PCRC(FileSystemProvider fs, String id) { + super(fs, id, FsInodeType.PCRC); + } + + @Override + public int read(long pos, byte[] data, int offset, int len) { + + if (_checksum == null) { + try { + _checksum = getChecksums(); + } catch (ChimeraFsException e) { + return -1; + } + } + + byte[] b = (_checksum).getBytes(); + + /* + * are we still inside ? + */ + if (pos > b.length) { + return 0; + } + + int copyLen = Math.min(len, b.length - (int) pos); + System.arraycopy(b, (int) pos, data, 0, copyLen); + + return copyLen; + } + + @Override + public Stat stat() throws ChimeraFsException { + + Stat ret = super.stat(); + ret.setMode((ret.getMode() & 0000777) | UnixPermission.S_IFREG); + if (_checksum == null) { + _checksum = getChecksums(); + } + + ret.setSize(_checksum.length()); + return ret; + } + + @Override + public int write(long pos, byte[] data, int offset, int len) { + return -1; + } + + private String getChecksums() throws ChimeraFsException { + Set results = _fs.getInodeChecksums(this); + StringBuilder sb = new StringBuilder(); + + Iterator it = results.iterator(); + if (it.hasNext()) { + Checksum result = it.next(); + sb.append(result.getType()) + .append(":") + .append(result.getValue()); + } + + while (it.hasNext()) { + Checksum result = it.next(); + sb.append(", ") + .append(result.getType()) + .append(":") + .append(result.getValue()); + } + + sb.append("\n"); + return sb.toString(); + } +} diff --git a/modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java b/modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java index ba3862f5ce2..adb944e4092 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/FsSqlDriver.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.UUID; import diskCacheV111.util.AccessLatency; @@ -44,6 +45,8 @@ import org.dcache.chimera.posix.Stat; import org.dcache.chimera.store.InodeStorageInformation; import org.dcache.commons.util.SqlHelper; +import org.dcache.util.Checksum; +import org.dcache.util.ChecksumType; /** * SQL driver @@ -2343,6 +2346,34 @@ String getInodeChecksum(Connection dbConnection, FsInode inode, int type) throws return checksum; } + private static final String sqlGetInodeChecksums = "SELECT isum, itype FROM t_inodes_checksum WHERE ipnfsid=?"; + /** + * + * @param dbConnection + * @param inode + * @param type + * @param results holds set of checksums and their types {@link Checksum} + * for this inode + * @throws SQLException + */ + void getInodeChecksums(Connection dbConnection, FsInode inode, Set results) + throws SQLException { + PreparedStatement stGetInodeChecksums = null; + ResultSet getGetInodeChecksumResultSet = null; + try { + stGetInodeChecksums = dbConnection.prepareStatement(sqlGetInodeChecksums); + stGetInodeChecksums.setString(1, inode.toString()); + getGetInodeChecksumResultSet = stGetInodeChecksums.executeQuery(); + if (getGetInodeChecksumResultSet.next()) { + String checksum = getGetInodeChecksumResultSet.getString("isum"); + int type = getGetInodeChecksumResultSet.getInt("itype"); + results.add(new Checksum(ChecksumType.getChecksumType(type), checksum)); + } + } finally { + SqlHelper.tryToClose(getGetInodeChecksumResultSet); + SqlHelper.tryToClose(stGetInodeChecksums); + } + } private static final String sqlRemoveInodeChecksum = "DELETE FROM t_inodes_checksum WHERE ipnfsid=? AND itype=?"; private static final String sqlRemoveInodeAllChecksum = "DELETE FROM t_inodes_checksum WHERE ipnfsid=?"; diff --git a/modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java b/modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java index ddb5e84e7c3..7e2edcf4938 100644 --- a/modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java +++ b/modules/chimera/src/main/java/org/dcache/chimera/JdbcFs.java @@ -28,7 +28,9 @@ import java.nio.ByteBuffer; import java.sql.Connection; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.StringTokenizer; import diskCacheV111.util.AccessLatency; @@ -37,6 +39,7 @@ import org.dcache.acl.ACE; import org.dcache.chimera.posix.Stat; import org.dcache.chimera.store.InodeStorageInformation; +import org.dcache.util.Checksum; import static org.dcache.commons.util.SqlHelper.tryToClose; @@ -1062,24 +1065,29 @@ public FsInode inodeOf(FsInode parent, String name) throws ChimeraFsException { throw new FileNotFoundHimeraFsException(name); } - if (cmd[2].equals("locality")) { - inode = inodeOf(parent, cmd[1]); - if (!inode.exists()) { - throw new FileNotFoundHimeraFsException(name); - } - return getPLOC(inode.toString()); - } - - /* - * pass in the name too (args 1 to n) - */ - String[] args = new String[cmd.length - 1]; - System.arraycopy(cmd, 1, args, 0, args.length); - inode = new FsInode_PGET(this, parent.toString(), args); - if (!inode.exists()) { - throw new FileNotFoundHimeraFsException(name); + switch(cmd[2]) { + case "locality": + inode = inodeOf(parent, cmd[1]); + if (!inode.exists()) { + throw new FileNotFoundHimeraFsException(name); + } + return getPLOC(inode.toString()); + case "checksum": + case "checksums": + inode = inodeOf(parent, cmd[1]); + if (!inode.exists()) { + throw new FileNotFoundHimeraFsException(name); + } + return new FsInode_PCRC(this, inode.toString()); + default: + String[] args = new String[cmd.length - 1]; + System.arraycopy(cmd, 1, args, 0, args.length); + inode = new FsInode_PGET(this, parent.toString(), args); + if (!inode.exists()) { + throw new FileNotFoundHimeraFsException(name); + } + return inode; } - return inode; } if (name.equals(".(config)")) { @@ -2527,6 +2535,28 @@ public String getInodeChecksum(FsInode inode, int type) throws ChimeraFsExceptio return checkSum; } + @Override + public Set getInodeChecksums(FsInode inode) throws ChimeraFsException { + Set checkSums = new HashSet<>(); + Connection dbConnection; + try { + // get from pool + dbConnection = _dbConnectionsPool.getConnection(); + } catch (SQLException e) { + throw new BackEndErrorHimeraFsException(e.getMessage()); + } + try { + dbConnection.setAutoCommit(true); + _sqlDriver.getInodeChecksums(dbConnection, inode, checkSums); + } catch (SQLException e) { + _log.error("getInodeChecksum", e); + throw new IOHimeraFsException(e.getMessage()); + } finally { + tryToClose(dbConnection); + } + return checkSums; + } + /** * Get inode's Access Control List. An empty list is returned if there are no ACL assigned * to the inode. @@ -2816,6 +2846,10 @@ FsInode inodeFromBytesNew(byte[] handle) throws ChimeraFsException { inode = getPLOC(inodeId); break; + case PCRC: + inode = new FsInode_PCRC(this, inodeId); + break; + default: throw new FileNotFoundHimeraFsException("Unsupported file handle type: " + inodeType); } @@ -2919,6 +2953,11 @@ FsInode inodeFromBytesOld(byte[] handle) throws ChimeraFsException { inode = getPLOC(id); break; + case PCRC: + id = st.nextToken(); + inode = new FsInode_PCRC(this, id); + break; + } } catch (IllegalArgumentException iae) { _log.info("Failed to generate an inode from file handle : {} : {}", strHandle, iae); diff --git a/modules/dcache/src/main/java/org/dcache/chimera/nfsv41/mover/DummyFileSystemProvider.java b/modules/dcache/src/main/java/org/dcache/chimera/nfsv41/mover/DummyFileSystemProvider.java index 8e573281e20..9a5d47d5bba 100644 --- a/modules/dcache/src/main/java/org/dcache/chimera/nfsv41/mover/DummyFileSystemProvider.java +++ b/modules/dcache/src/main/java/org/dcache/chimera/nfsv41/mover/DummyFileSystemProvider.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Set; import diskCacheV111.util.AccessLatency; import diskCacheV111.util.RetentionPolicy; @@ -18,6 +19,7 @@ import org.dcache.chimera.StorageLocatable; import org.dcache.chimera.posix.Stat; import org.dcache.chimera.store.InodeStorageInformation; +import org.dcache.util.Checksum; public class DummyFileSystemProvider implements FileSystemProvider { @@ -143,6 +145,14 @@ public String getInodeChecksum(FsInode arg0, int arg1) return null; } + @Override + public Set getInodeChecksums(FsInode inode) + throws ChimeraFsException { + // TODO Auto-generated method stub + return null; + } + + @Override public List getInodeLocations(FsInode arg0, int arg1) throws ChimeraFsException {