feat: Add bloom filter related proto fields (#1843)

* feat: Add bloom filter related proto fields (only in the preview API surface)

PiperOrigin-RevId: 527090049

Source-Link: googleapis/googleapis@e2b7cb9

Source-Link: googleapis/googleapis-gen@b0d2cc1
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiYjBkMmNjMWM0OGRkYWMxYzVkYmFjMWNlMTk5ZDI5ZWFmMWM1ZWMwYyJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* feat: Add bloom filter related proto fields

PiperOrigin-RevId: 529511263

Source-Link: googleapis/googleapis@b071320

Source-Link: googleapis/googleapis-gen@81dcde7
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiODFkY2RlNzA4YTZlMzlkYTBmMmY0N2RjOGYxNmVlNWU2ODFhNTU5ZiJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

dev/protos/google/api/client.proto

@@ -157,7 +157,7 @@ message Publishing {
   // long-running operation pattern.
   repeated MethodSettings method_settings = 2;
 
-  // Link to a place that API users can report issues.  Example:
+  // Link to a *public* URI where users can report issues.  Example:
   // https://issuetracker.google.com/issues/new?component=190865&template=1161103
   string new_issue_uri = 101;
 
@@ -368,6 +368,15 @@ enum ClientLibraryOrganization {
 
   // Street View Org.
   STREET_VIEW = 4;
+
+  // Shopping Org.
+  SHOPPING = 5;
+
+  // Geo Org.
+  GEO = 6;
+
+  // Generative AI - https://developers.generativeai.google
+  GENERATIVE_AI = 7;
 }
 
 // To where should client libraries be published?

dev/protos/google/firestore/v1/aggregation_result.proto

@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

dev/protos/google/firestore/v1/bloom_filter.proto

@@ -0,0 +1,73 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto3";
+
+package google.firestore.v1;
+
+option csharp_namespace = "Google.Cloud.Firestore.V1";
+option go_package = "cloud.google.com/go/firestore/apiv1/firestorepb;firestorepb";
+option java_multiple_files = true;
+option java_outer_classname = "BloomFilterProto";
+option java_package = "com.google.firestore.v1";
+option objc_class_prefix = "GCFS";
+option php_namespace = "Google\\Cloud\\Firestore\\V1";
+option ruby_package = "Google::Cloud::Firestore::V1";
+
+// A sequence of bits, encoded in a byte array.
+//
+// Each byte in the `bitmap` byte array stores 8 bits of the sequence. The only
+// exception is the last byte, which may store 8 _or fewer_ bits. The `padding`
+// defines the number of bits of the last byte to be ignored as "padding". The
+// values of these "padding" bits are unspecified and must be ignored.
+//
+// To retrieve the first bit, bit 0, calculate: `(bitmap[0] & 0x01) != 0`.
+// To retrieve the second bit, bit 1, calculate: `(bitmap[0] & 0x02) != 0`.
+// To retrieve the third bit, bit 2, calculate: `(bitmap[0] & 0x04) != 0`.
+// To retrieve the fourth bit, bit 3, calculate: `(bitmap[0] & 0x08) != 0`.
+// To retrieve bit n, calculate: `(bitmap[n / 8] & (0x01 << (n % 8))) != 0`.
+//
+// The "size" of a `BitSequence` (the number of bits it contains) is calculated
+// by this formula: `(bitmap.length * 8) - padding`.
+message BitSequence {
+  // The bytes that encode the bit sequence.
+  // May have a length of zero.
+  bytes bitmap = 1;
+
+  // The number of bits of the last byte in `bitmap` to ignore as "padding".
+  // If the length of `bitmap` is zero, then this value must be `0`.
+  // Otherwise, this value must be between 0 and 7, inclusive.
+  int32 padding = 2;
+}
+
+// A bloom filter (https://en.wikipedia.org/wiki/Bloom_filter).
+//
+// The bloom filter hashes the entries with MD5 and treats the resulting 128-bit
+// hash as 2 distinct 64-bit hash values, interpreted as unsigned integers
+// using 2's complement encoding.
+//
+// These two hash values, named `h1` and `h2`, are then used to compute the
+// `hash_count` hash values using the formula, starting at `i=0`:
+//
+//     h(i) = h1 + (i * h2)
+//
+// These resulting values are then taken modulo the number of bits in the bloom
+// filter to get the bits of the bloom filter to test for the given entry.
+message BloomFilter {
+  // The bloom filter data.
+  BitSequence bits = 1;
+
+  // The number of hashes used by the algorithm.
+  int32 hash_count = 2;
+}

dev/protos/google/firestore/v1/common.proto

@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

dev/protos/google/firestore/v1/document.proto

@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

dev/protos/google/firestore/v1/firestore.proto

@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -26,6 +26,7 @@ import "google/firestore/v1/query.proto";
 import "google/firestore/v1/write.proto";
 import "google/protobuf/empty.proto";
 import "google/protobuf/timestamp.proto";
+import "google/protobuf/wrappers.proto";
 import "google/rpc/status.proto";
 
 option csharp_namespace = "Google.Cloud.Firestore.V1";
@@ -178,16 +179,16 @@ service Firestore {
   }
 
   // Streams batches of document updates and deletes, in order. This method is
-  // only available via the gRPC API (not REST).
+  // only available via gRPC or WebChannel (not REST).
   rpc Write(stream WriteRequest) returns (stream WriteResponse) {
     option (google.api.http) = {
       post: "/v1/{database=projects/*/databases/*}/documents:write"
       body: "*"
     };
   }
 
-  // Listens to changes. This method is only available via the gRPC API (not
-  // REST).
+  // Listens to changes. This method is only available via gRPC or WebChannel
+  // (not REST).
   rpc Listen(stream ListenRequest) returns (stream ListenResponse) {
     option (google.api.http) = {
       post: "/v1/{database=projects/*/databases/*}/documents:listen"
@@ -655,7 +656,14 @@ message RunAggregationQueryResponse {
   // a new transaction.
   bytes transaction = 2;
 
-  // The time at which the aggregate value is valid for.
+  // The time at which the aggregate result was computed. This is always
+  // monotonically increasing; in this case, the previous AggregationResult in
+  // the result stream are guaranteed not to have changed between their
+  // `read_time` and this one.
+  //
+  // If the query returns no results, a response with `read_time` and no
+  // `result` will be sent, and this represents the time at which the query
+  // was run.
   google.protobuf.Timestamp read_time = 3;
 }
 
@@ -925,6 +933,14 @@ message Target {
 
   // If the target should be removed once it is current and consistent.
   bool once = 6;
+
+  // The number of documents that last matched the query at the resume token or
+  // read time.
+  //
+  // This value is only relevant when a `resume_type` is provided. This value
+  // being present and greater than zero signals that the client wants
+  // `ExistenceFilter.unchanged_names` to be included in the response.
+  google.protobuf.Int32Value expected_count = 12;
 }
 
 // Targets being watched have changed.

dev/protos/google/firestore/v1/query.proto

@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -136,25 +136,28 @@ message StructuredQuery {
       //
       // Requires:
       //
-      // * That `value` is a non-empty `ArrayValue` with at most 10 values.
-      // * No other `IN` or `ARRAY_CONTAINS_ANY` or `NOT_IN`.
+      // * That `value` is a non-empty `ArrayValue`, subject to disjunction
+      //   limits.
+      // * No `NOT_IN` filters in the same query.
       IN = 8;
 
       // The given `field` is an array that contains any of the values in the
       // given array.
       //
       // Requires:
       //
-      // * That `value` is a non-empty `ArrayValue` with at most 10 values.
-      // * No other `IN` or `ARRAY_CONTAINS_ANY` or `NOT_IN`.
+      // * That `value` is a non-empty `ArrayValue`, subject to disjunction
+      //   limits.
+      // * No other `ARRAY_CONTAINS_ANY` filters within the same disjunction.
+      // * No `NOT_IN` filters in the same query.
       ARRAY_CONTAINS_ANY = 9;
 
       // The value of the `field` is not in the given array.
       //
       // Requires:
       //
       // * That `value` is a non-empty `ArrayValue` with at most 10 values.
-      // * No other `IN`, `ARRAY_CONTAINS_ANY`, `NOT_IN`, `NOT_EQUAL`,
+      // * No other `OR`, `IN`, `ARRAY_CONTAINS_ANY`, `NOT_IN`, `NOT_EQUAL`,
       //   `IS_NOT_NULL`, or `IS_NOT_NAN`.
       // * That `field` comes first in the `order_by`.
       NOT_IN = 10;
@@ -251,7 +254,11 @@ message StructuredQuery {
     repeated FieldReference fields = 2;
   }
 
-  // The projection to return.
+  // Optional sub-set of the fields to return.
+  //
+  // This acts as a [DocumentMask][google.firestore.v1.DocumentMask] over the
+  // documents returned from a query. When not set, assumes that the caller
+  // wants all fields returned.
   Projection select = 1;
 
   // The collections to query.
@@ -349,7 +356,7 @@ message StructuredQuery {
 // Firestore query for running an aggregation over a
 // [StructuredQuery][google.firestore.v1.StructuredQuery].
 message StructuredAggregationQuery {
-  // Defines a aggregation that produces a single result.
+  // Defines an aggregation that produces a single result.
   message Aggregation {
     // Count of documents that match the query.
     //
@@ -360,7 +367,7 @@ message StructuredAggregationQuery {
       // count.
       //
       // This provides a way to set an upper bound on the number of documents
-      // to scan, limiting latency and cost.
+      // to scan, limiting latency, and cost.
       //
       // Unspecified is interpreted as no bound.
       //
@@ -394,7 +401,7 @@ message StructuredAggregationQuery {
     //   COUNT_UP_TO(1) AS count_up_to_1,
     //   COUNT_UP_TO(2),
     //   COUNT_UP_TO(3) AS count_up_to_3,
-    //   COUNT_UP_TO(4)
+    //   COUNT(*)
     // OVER (
     //   ...
     // );
@@ -407,7 +414,7 @@ message StructuredAggregationQuery {
     //   COUNT_UP_TO(1) AS count_up_to_1,
     //   COUNT_UP_TO(2) AS field_1,
     //   COUNT_UP_TO(3) AS count_up_to_3,
-    //   COUNT_UP_TO(4) AS field_2
+    //   COUNT(*) AS field_2
     // OVER (
     //   ...
     // );

dev/protos/google/firestore/v1/write.proto

@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@ syntax = "proto3";
 
 package google.firestore.v1;
 
+import "google/firestore/v1/bloom_filter.proto";
 import "google/firestore/v1/common.proto";
 import "google/firestore/v1/document.proto";
 import "google/protobuf/timestamp.proto";
@@ -264,4 +265,19 @@ message ExistenceFilter {
   // If different from the count of documents in the client that match, the
   // client must manually determine which documents no longer match the target.
   int32 count = 2;
+
+  // A bloom filter that contains the UTF-8 byte encodings of the resource names
+  // of the documents that match
+  // [target_id][google.firestore.v1.ExistenceFilter.target_id], in the form
+  // `projects/{project_id}/databases/{database_id}/documents/{document_path}`
+  // that have NOT changed since the query results indicated by the resume token
+  // or timestamp given in `Target.resume_type`.
+  //
+  // This bloom filter may be omitted at the server's discretion, such as if it
+  // is deemed that the client will not make use of it or if it is too
+  // computationally expensive to calculate or transmit. Clients must gracefully
+  // handle this field being absent by falling back to the logic used before
+  // this field existed; that is, re-add the target without a resume token to
+  // figure out which documents in the client's cache are out of sync.
+  BloomFilter unchanged_names = 3;
 }

dev/protos/google/protobuf/descriptor.proto

@@ -146,14 +146,17 @@ message ExtensionRangeOptions {
     // and enums.
     optional string type = 3;
 
-    // If true, indicates that the extension must be defined as repeated.
-    // Otherwise the extension must be defined as optional.
-    optional bool is_repeated = 4;
+    // Deprecated. Please use "repeated".
+    optional bool is_repeated = 4 [deprecated = true];
 
     // If true, indicates that the number is reserved in the extension range,
     // and any extension field with the number will fail to compile. Set this
     // when a declared extension field is deleted.
     optional bool reserved = 5;
+
+    // If true, indicates that the extension must be defined as repeated.
+    // Otherwise the extension must be defined as optional.
+    optional bool repeated = 6;
   }
 
   // go/protobuf-stripping-extension-declarations
@@ -162,6 +165,18 @@ message ExtensionRangeOptions {
   // extension range into small ranges in generated binaries.
   repeated Declaration declaration = 2 [retention = RETENTION_SOURCE];
 
+  // The verification state of the extension range.
+  enum VerificationState {
+    // All the extensions of the range must be declared.
+    DECLARATION = 0;
+    UNVERIFIED = 1;
+  }
+
+  // The verification state of the range.
+  // TODO(b/278783756): flip the default to DECLARATION once all empty ranges
+  // are marked as UNVERIFIED.
+  optional VerificationState verification = 3 [default = UNVERIFIED];
+
   // Clients can define custom options in extensions of this message. See above.
   extensions 1000 to max;
 }
@@ -569,13 +584,21 @@ message MessageOptions {
 message FieldOptions {
   // The ctype option instructs the C++ code generator to use a different
   // representation of the field than it normally would.  See the specific
-  // options below.  This option is not yet implemented in the open source
-  // release -- sorry, we'll try to include it in a future version!
+  // options below.  This option is only implemented to support use of
+  // [ctype=CORD] and [ctype=STRING] (the default) on non-repeated fields of
+  // type "bytes" in the open source release -- sorry, we'll try to include
+  // other types in a future version!
   optional CType ctype = 1 [default = STRING];
   enum CType {
     // Default mode.
     STRING = 0;
 
+    // The option [ctype=CORD] may be applied to a non-repeated field of type
+    // "bytes". It indicates that in C++, the data should be stored in a Cord
+    // instead of a string.  For very large strings, this may reduce memory
+    // fragmentation. It may also allow better performance when parsing from a
+    // Cord, or when parsing with aliasing enabled, as the parsed Cord may then
+    // alias the original buffer.
     CORD = 1;
 
     STRING_PIECE = 2;
@@ -688,7 +711,6 @@ message FieldOptions {
     TARGET_TYPE_METHOD = 9;
   }
 
-  optional OptionTargetType target = 18 [deprecated = true];
   repeated OptionTargetType targets = 19;
 
   // The parser stores options it doesn't recognize here. See above.
@@ -698,9 +720,11 @@ message FieldOptions {
   extensions 1000 to max;
 
   reserved 4;  // removed jtype
+  optional OptionTargetType target_obsolete_do_not_use = 18 [deprecated = true];
 }
 
 message OneofOptions {
+
   // The parser stores options it doesn't recognize here. See above.
   repeated UninterpretedOption uninterpreted_option = 999;