PARQUET-1477: Thrift crypto updates (#124)

src/main/thrift/parquet.thrift

@@ -670,8 +670,8 @@ struct EncryptionWithColumnKey {
   /** Column path in schema **/
   1: required list<string> path_in_schema
 
-  /** Retrieval metadata of the column-specific key **/
-  2: optional binary column_key_metadata
+  /** Retrieval metadata of column encryption key **/
+  2: optional binary key_metadata
 }
 
 union ColumnCryptoMetaData {
@@ -707,7 +707,10 @@ struct ColumnChunk {
   7: optional i32 column_index_length
 
   /** Crypto metadata of encrypted columns **/
-  8: optional ColumnCryptoMetaData crypto_meta_data
+  8: optional ColumnCryptoMetaData crypto_metadata
+
+  /** Encrypted column metadata for this chunk **/
+  9: optional binary encrypted_column_metadata
 }
 
 struct RowGroup {
@@ -734,6 +737,9 @@ struct RowGroup {
   /** Total byte size of all compressed (and potentially encrypted) column data 
    *  in this row group **/
   6: optional i64 total_compressed_size
+
+  /** Row group ordinal in the file **/
+  7: optional i16 ordinal
 }
 
 /** Empty struct to signal the order defined by the physical or logical type */
@@ -863,23 +869,27 @@ struct ColumnIndex {
 }
 
 struct AesGcmV1 {
-  /** Retrieval metadata of AAD used for encryption of pages and structures **/
-  1: optional binary aad_metadata
+  /** AAD prefix **/
+  1: optional binary aad_prefix
 
-  /** If file IVs are comprised of a fixed part, and variable parts
-   *  (e.g. counter), keep the fixed part here **/
-  2: optional binary iv_prefix
+  /** Unique file identifier part of AAD suffix **/
+  2: optional binary aad_file_unique
+
+  /** In files encrypted with AAD prefix without storing it,
+   * readers must supply the prefix **/
+  3: optional bool supply_aad_prefix
 }
 
 struct AesGcmCtrV1 {
-  /** Retrieval metadata of AAD used for encryption of structures **/
-  1: optional binary aad_metadata
-
-  /** If file IVs are comprised of a fixed part, and variable parts
-   *  (e.g. counter), keep the fixed part here **/
-  2: optional binary gcm_iv_prefix
+  /** AAD prefix **/
+  1: optional binary aad_prefix
 
-  3: optional binary ctr_iv_prefix
+  /** Unique file identifier part of AAD suffix **/
+  2: optional binary aad_file_unique
+
+  /** In files encrypted with AAD prefix without storing it,
+   * readers must supply the prefix **/
+  3: optional bool supply_aad_prefix
 }
 
 union EncryptionAlgorithm {
@@ -932,27 +942,30 @@ struct FileMetaData {
   7: optional list<ColumnOrder> column_orders
 
   /** 
-   * Encryption algorithm. Note that this field is only used for files
-   * with plaintext footer. Files with encrypted footer store the algorithm id
+   * Encryption algorithm. This field is set only in encrypted files
+   * with plaintext footer. Files with encrypted footer store algorithm id
    * in FileCryptoMetaData structure.
    */
   8: optional EncryptionAlgorithm encryption_algorithm
+
+  /** 
+   * Retrieval metadata of key used for signing the footer. 
+   * Used only in encrypted files with plaintext footer. 
+   */ 
+  9: optional binary footer_signing_key_metadata
 }
 
 /** Crypto metadata for files with encrypted footer **/
 struct FileCryptoMetaData {
   /** 
-   * Encryption algorithm. Note that this field is only used for files
-   * with encrypted footer. Files with plaintext footer store the algorithm id
+   * Encryption algorithm. This field is only used for files
+   * with encrypted footer. Files with plaintext footer store algorithm id
    * inside footer (FileMetaData structure).
    */
   1: required EncryptionAlgorithm encryption_algorithm
 
   /** Retrieval metadata of key used for encryption of footer, 
    *  and (possibly) columns **/
-  2: optional binary footer_key_metadata
-
-  /** Offset of encrypted Parquet footer **/
-  3: required i64 footer_offset
+  2: optional binary key_metadata
 }