Compute sha1 sum for uncompressed data file

patch by slebresne; reviewed by jbellis for CASSANDRA-3456


git-svn-id: https://svn.apache.org/repos/asf/cassandra/branches/cassandra-1.0@1199763 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -4,6 +4,7 @@
  * (CQL) fix handling of rows with no columns (CASSANDRA-3424, 3473)
  * fix querying supercolumns by name returning only a subset of
    subcolumns or old subcolumn versions (CASSANDRA-3446)
+ * automatically compute sha1 sum for uncompressed data files (CASSANDRA-3456)
 Merged from 0.8:
  * Make counter shard merging thread safe (CASSANDRA-3178)
  * fix updating CF row_cache_provider (CASSANDRA-3414)

src/java/org/apache/cassandra/io/sstable/Component.java

@@ -54,7 +54,9 @@ enum Type
         // statistical metadata about the content of the sstable
         STATS("Statistics.db"),
         // a bitmap secondary index: many of these may exist per sstable
-        BITMAP_INDEX("Bitidx.db");
+        BITMAP_INDEX("Bitidx.db"),
+        // holds sha1 sum of the data file (to be checked by sha1sum)
+        DIGEST("Digest.sha1");
 
         final String repr;
         Type(String repr)
@@ -78,6 +80,7 @@ static Type fromRepresentation(String repr)
     public final static Component COMPACTED_MARKER = new Component(Type.COMPACTED_MARKER, -1);
     public final static Component COMPRESSION_INFO = new Component(Type.COMPRESSION_INFO, -1);
     public final static Component STATS = new Component(Type.STATS, -1);
+    public final static Component DIGEST = new Component(Type.DIGEST, -1);
 
     public final Type type;
     public final int id;
@@ -108,6 +111,7 @@ public String name()
             case COMPACTED_MARKER:
             case COMPRESSION_INFO:
             case STATS:
+            case DIGEST:
                 return type.repr;
             case BITMAP_INDEX:
                 return String.format("%d%c%s", id, separator, type.repr);
@@ -145,6 +149,7 @@ public static Pair<Descriptor,Component> fromFilename(File directory, String nam
             case COMPACTED_MARKER:  component = Component.COMPACTED_MARKER; break;
             case COMPRESSION_INFO:  component = Component.COMPRESSION_INFO; break;
             case STATS:             component = Component.STATS;            break;
+            case DIGEST:            component = Component.DIGEST;           break;
             case BITMAP_INDEX:
                  component = new Component(type, id);
                  break;

src/java/org/apache/cassandra/io/sstable/SSTable.java

@@ -58,6 +58,7 @@ public abstract class SSTable
     public static final String COMPONENT_INDEX = Component.Type.PRIMARY_INDEX.repr;
     public static final String COMPONENT_FILTER = Component.Type.FILTER.repr;
     public static final String COMPONENT_STATS = Component.Type.STATS.repr;
+    public static final String COMPONENT_DIGEST = Component.Type.DIGEST.repr;
 
     public static final String TEMPFILE_MARKER = "tmp";
 

src/java/org/apache/cassandra/io/sstable/SSTableWriter.java

@@ -37,9 +37,7 @@
 import org.apache.cassandra.io.compress.CompressedSequentialWriter;
 import org.apache.cassandra.io.util.*;
 import org.apache.cassandra.service.StorageService;
-import org.apache.cassandra.utils.BloomFilter;
-import org.apache.cassandra.utils.ByteBufferUtil;
-import org.apache.cassandra.utils.FBUtilities;
+import org.apache.cassandra.utils.*;
 
 public class SSTableWriter extends SSTable
 {
@@ -66,6 +64,10 @@ private static Set<Component> components(CFMetaData metadata)
         Set<Component> components = new HashSet<Component>(Arrays.asList(Component.DATA, Component.FILTER, Component.PRIMARY_INDEX, Component.STATS));
         if (metadata.compressionParameters().sstableCompressor != null)
             components.add(Component.COMPRESSION_INFO);
+        else
+            // it would feel safer to actually add this component later in maybeWriteDigest(),
+            // but the components are unmodifiable after construction
+            components.add(Component.DIGEST);
         return components;
     }
 
@@ -94,6 +96,7 @@ public SSTableWriter(String filename,
         {
             dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode());
             dataFile = SequentialWriter.open(new File(getFilename()), true);
+            dataFile.setComputeDigest();
         }
 
         this.sstableMetadataCollector = sstableMetadataCollector;
@@ -300,6 +303,7 @@ public SSTableReader closeAndOpenReader(long maxDataAge) throws IOException
         // write sstable statistics
         SSTableMetadata sstableMetadata = sstableMetadataCollector.finalizeMetadata();
         writeMetadata(descriptor, sstableMetadata);
+        maybeWriteDigest();
 
         // remove the 'tmp' marker from all components
         final Descriptor newdesc = rename(descriptor, components);
@@ -324,6 +328,21 @@ public SSTableReader closeAndOpenReader(long maxDataAge) throws IOException
         return sstable;
     }
 
+    private void maybeWriteDigest() throws IOException
+    {
+        byte[] digest = dataFile.digest();
+        if (digest == null)
+            return;
+
+        SequentialWriter out = SequentialWriter.open(new File(descriptor.filenameFor(SSTable.COMPONENT_DIGEST)), true);
+        // Writting output compatible with sha1sum
+        Descriptor newdesc = descriptor.asTemporary(false);
+        String[] tmp = newdesc.filenameFor(SSTable.COMPONENT_DATA).split(new Character(File.separatorChar).toString());
+        String dataFileName = tmp[tmp.length - 1];
+        out.write(String.format("%s  %s", Hex.bytesToHex(digest), dataFileName).getBytes());
+        out.close();
+    }
+
     private static void writeMetadata(Descriptor desc, SSTableMetadata sstableMetadata) throws IOException
     {
         SequentialWriter out = SequentialWriter.open(new File(desc.filenameFor(SSTable.COMPONENT_STATS)), true);

src/java/org/apache/cassandra/io/util/SequentialWriter.java

@@ -20,6 +20,8 @@
 
 import java.io.*;
 import java.nio.channels.ClosedChannelException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 
 import org.apache.cassandra.utils.CLibrary;
 
@@ -47,6 +49,7 @@ public class SequentialWriter extends OutputStream
     private long ioCacheStartOffset = 0, bytesSinceCacheFlush = 0;
 
     public final DataOutputStream stream;
+    private MessageDigest digest;
 
     public SequentialWriter(File file, int bufferSize, boolean skipIOCache) throws IOException
     {
@@ -199,6 +202,8 @@ protected void flushInternal() throws IOException
     protected void flushData() throws IOException
     {
         out.write(buffer, 0, validBufferBytes);
+        if (digest != null)
+            digest.update(buffer, 0, validBufferBytes);
     }
 
     public long getFilePointer()
@@ -285,6 +290,41 @@ public void close() throws IOException
         out.close();
     }
 
+    /**
+     * Turn on digest computation on this writer.
+     * This can only be called before any data is written to this write,
+     * otherwise an IllegalStateException is thrown.
+     */
+    public void setComputeDigest()
+    {
+        if (current != 0)
+            throw new IllegalStateException();
+
+        try
+        {
+            digest = MessageDigest.getInstance("SHA-1");
+        }
+        catch (NoSuchAlgorithmException e)
+        {
+            // SHA-1 is standard in java 6
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Return the digest associated to this file or null if no digest was
+     * created.
+     * This can only be called once the file is fully created, i.e. after
+     * close() has been called. Otherwise an IllegalStateException is thrown.
+     */
+    public byte[] digest()
+    {
+        if (buffer != null)
+            throw new IllegalStateException();
+
+        return digest == null ? null : digest.digest();
+    }
+
     /**
      * Class to hold a mark to the position of the file
      */