safebrowsing.proto

// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

import "google/protobuf/duration.proto";

package safebrowsing_proto;

// The Safe Browsing APIs enable clients to check web resources (most commonly
// URLs) against Google's constantly updated lists of unsafe web resources. The
// Safe Browsing APIs (v4) feature the Update API and the Lookup API.
//
// The Update API is designed for our larger clients and includes the
// FindFullHashes and FetchThreatListUpdates methods. The Update API requires
// that clients maintain downloaded threat lists in a local database.
//
// First, clients match against their local lists to determine the state (safe
// or unsafe) of a given web resource. Most commonly, lists are comprised of
// hash prefixes of blacklisted URL expressions. To check a URL, clients
// generate the hashes of a given URL and check for prefix collisions in their
// local lists; if a prefix match is found, the client obtains the full hashes
// associated with the matched hash prefix via the FindFullHashes method. The
// client then compares the local full hash with the returned full hashes; a match
// indicates that the URL is unsafe.
//
// Second, clients obtain updates to their local databases via the
// FetchThreatListUpdates method, which takes the current state of the client
// and returns an updated client state along with the changes that the client
// should apply to their local threat lists.
//
// The Lookup API is designed for our smaller clients and allows them to match
// resources directly against the Safe Browsing threat lists via the
// FindThreatMatches method.
//
// Clients using either the Update API or the Lookup API can obtain a list of
// the Safe Browsing threat lists available for download via the ListThreatLists
// method.

// The information regarding one or more threats that a client submits when
// checking for matches in threat lists.
message ThreatInfo {
  // The threat types to be checked.
  repeated ThreatType threat_types = 1;

  // The platform types to be checked.
  repeated PlatformType platform_types = 2;

  // The entry types to be checked.
  repeated ThreatEntryType threat_entry_types = 4;

  // The threat entries to be checked.
  repeated ThreatEntry threat_entries = 3;
}

// A match when checking a threat entry in the Safe Browsing threat lists.
message ThreatMatch {
  // The threat type matching this threat.
  ThreatType threat_type = 1;

  // The platform type matching this threat.
  PlatformType platform_type = 2;

  // The threat entry type matching this threat.
  ThreatEntryType threat_entry_type = 6;

  // The threat matching this threat.
  ThreatEntry threat = 3;

  // Optional metadata associated with this threat.
  ThreatEntryMetadata threat_entry_metadata = 4;

  // The cache lifetime for the returned match. Clients must not cache this
  // response for more than this duration to avoid false positives.
  google.protobuf.Duration cache_duration = 5;
}

// Request to check entries against lists.
message FindThreatMatchesRequest {
  // The client metadata.
  ClientInfo client = 1;

  // The lists and entries to be checked for matches.
  ThreatInfo threat_info = 2;
}

// Response type for requests to find threat matches.
message FindThreatMatchesResponse {
  // The threat list matches.
  repeated ThreatMatch matches = 1;
}

// Describes a Safe Browsing API update request. Clients can request updates for
// multiple lists in a single request.
// NOTE: Field index 2 is unused.
message FetchThreatListUpdatesRequest {
  // The client metadata.
  ClientInfo client = 1;

  // A single list update request.
  message ListUpdateRequest {
    // The type of threat posed by entries present in the list.
    ThreatType threat_type = 1;

    // The type of platform at risk by entries present in the list.
    PlatformType platform_type = 2;

    // The types of entries present in the list.
    ThreatEntryType threat_entry_type = 5;

    // The current state of the client for the requested list (the encrypted
    // ClientState that was sent to the client from the previous update
    // request).
    bytes state = 3;

    // The constraints for this update.
    message Constraints {
      // The maximum size in number of entries. The update will not contain more
      // entries than this value.  This should be a power of 2 between 2**10 and
      // 2**20.  If zero, no update size limit is set.
      int32 max_update_entries = 1;

      // Sets the maximum number of entries that the client is willing to have
      // in the local database. This should be a power of 2 between 2**10 and
      // 2**20. If zero, no database size limit is set.
      int32 max_database_entries = 2;

      // Requests the list for a specific geographic location. If not set the
      // server may pick that value based on the user's IP address. Expects ISO
      // 3166-1 alpha-2 format.
      string region = 3;

      // The compression types supported by the client.
      repeated CompressionType supported_compressions = 4;
    }
 
    // The constraints associated with this request.
    Constraints constraints = 4;
  }

   // Index 2 is unused.

  // The requested threat list updates.
  repeated ListUpdateRequest list_update_requests = 3;
}

// Response type for threat list update requests.
message FetchThreatListUpdatesResponse {
  // An update to an individual list.
  message ListUpdateResponse {
    // The threat type for which data is returned.
    ThreatType threat_type = 1;

    // The format of the threats.
    ThreatEntryType threat_entry_type = 2;

    // The platform type for which data is returned.
    PlatformType platform_type = 3;

    // The type of response sent to the client.
    enum ResponseType {
      // Unknown.
      RESPONSE_TYPE_UNSPECIFIED = 0;

      // Partial updates are applied to the client's existing local database.
      PARTIAL_UPDATE = 1;

      // Full updates replace the client's entire local database. This means
      // that either the client was seriously out-of-date or the client is
      // believed to be corrupt.
      FULL_UPDATE = 2;
    }

    // The type of response. This may indicate that an action is required by the
    // client when the response is received.
    ResponseType response_type = 4;

    // A set of entries to add to a local threat type's list. Repeated to allow
    // for a combination of compressed and raw data to be sent in a single
    // response.
    repeated ThreatEntrySet additions = 5;

    // A set of entries to remove from a local threat type's list. Repeated for
    // the same reason as above.
    repeated ThreatEntrySet removals = 6;

    // The new client state, in encrypted format. Opaque to clients.
    bytes new_client_state = 7;

    // The expected SHA256 hash of the client state; that is, of the sorted list
    // of all hashes present in the database after applying the provided update.
    // If the client state doesn't match the expected state, the client must
    // disregard this update and retry later.
    Checksum checksum = 8;
  }

  // The list updates requested by the clients.
  repeated ListUpdateResponse list_update_responses = 1;

  // The minimum duration the client must wait before issuing any update
  // request. If this field is not set clients may update as soon as they want.
  google.protobuf.Duration minimum_wait_duration = 2;
}

// Request to return full hashes matched by the provided hash prefixes.
message FindFullHashesRequest {
  // The client metadata.
  ClientInfo client = 1;

  // The current client states for each of the client's local threat lists.
  repeated bytes client_states = 2;

  // The lists and hashes to be checked.
  ThreatInfo threat_info = 3;
}

// Response type for requests to find full hashes.
message FindFullHashesResponse {
  // The full hashes that matched the requested prefixes.
  repeated ThreatMatch matches = 1;

  // The minimum duration the client must wait before issuing any find hashes
  // request. If this field is not set, clients can issue a request as soon as
  // they want.
  google.protobuf.Duration minimum_wait_duration = 2;

  // For requested entities that did not match the threat list, how long to
  // cache the response.
  google.protobuf.Duration negative_cache_duration = 3;
}


// Types of threats.
enum ThreatType {
  // Unknown.
  THREAT_TYPE_UNSPECIFIED = 0;

  // Malware threat type.
  MALWARE = 1;

  // Social engineering threat type.
  SOCIAL_ENGINEERING = 2;

  // Unwanted software threat type.
  UNWANTED_SOFTWARE = 3;

  // Potentially harmful application threat type.
  POTENTIALLY_HARMFUL_APPLICATION = 4;
}

// Types of platforms.
enum PlatformType {
  // Unknown platform.
  PLATFORM_TYPE_UNSPECIFIED = 0;

  // Threat posed to Windows.
  WINDOWS = 1;

  // Threat posed to Linux.
  LINUX = 2;

  // Threat posed to Android.
  ANDROID = 3;

  // Threat posed to OSX.
  OSX = 4;

  // Threat posed to iOS.
  IOS = 5;

  // Threat posed to at least one of the defined platforms.
  ANY_PLATFORM = 6;

  // Threat posed to all defined platforms.
  ALL_PLATFORMS = 7;

  // Threat posed to Chrome.
  CHROME = 8;
}

// The client metadata associated with Safe Browsing API requests.
message ClientInfo {
  // A client ID that (hopefully) uniquely identifies the client implementation
  // of the Safe Browsing API.
  string client_id = 1;

  // The version of the client implementation.
  string client_version = 2;
}

// The expected state of a client's local database.
message Checksum {
  // The SHA256 hash of the client state; that is, of the sorted list of all
  // hashes present in the database.
  bytes sha256 = 1;
}

// The ways in which threat entry sets can be compressed.
enum CompressionType {
  // Unknown.
  COMPRESSION_TYPE_UNSPECIFIED = 0;

  // Raw, uncompressed data.
  RAW = 1;

  // Rice-Golomb encoded data.
  RICE = 2;
}

// An individual threat; for example, a malicious URL or its hash
// representation. Only one of these fields should be set.
message ThreatEntry {
  // A hash prefix, consisting of the most significant 4-32 bytes of a SHA256
  // hash.
  bytes hash = 1;

  // A URL.
  string url = 2;
}

// Types of entries that pose threats. Threat lists are collections of entries
// of a single type.
enum ThreatEntryType {
  // Unspecified.
  THREAT_ENTRY_TYPE_UNSPECIFIED = 0;

  // A URL.
  URL = 1;

  // An executable program.
  EXECUTABLE = 2;

  // An IP range.
  IP_RANGE = 3;
}

// A set of threats that should be added or removed from a client's local
// database.
message ThreatEntrySet {
  // The compression type for the entries in this set.
  CompressionType compression_type = 1;

  // At most one of the following fields should be set.

  // The raw SHA256-formatted entries.
  RawHashes raw_hashes = 2;

  // The raw removal indices for a local list.
  RawIndices raw_indices = 3;

  // The encoded 4-byte prefixes of SHA256-formatted entries, using a
  // Golomb-Rice encoding.
  RiceDeltaEncoding rice_hashes = 4;

  // The encoded local, lexicographically-sorted list indices, using a
  // Golomb-Rice encoding. Used for sending compressed removal indices.
  RiceDeltaEncoding rice_indices = 5;
}

// A set of raw indices to remove from a local list.
message RawIndices {
  // The indices to remove from a lexicographically-sorted local list.
  repeated int32 indices = 1;
}

// The uncompressed threat entries in hash format of a particular prefix length.
// Hashes can be anywhere from 4 to 32 bytes in size. A large majority are 4
// bytes, but some hashes are lengthened if they collide with the hash of a
// popular URL.
//
// Used for sending ThreatEntrySet to clients that do not support compression,
// or when sending non-4-byte hashes to clients that do support compression.
message RawHashes {
  // The number of bytes for each prefix encoded below.  This field can be
  // anywhere from 4 (shortest prefix) to 32 (full SHA256 hash).
  int32 prefix_size = 1;

  // The hashes, all concatenated into one long string.  Each hash has a prefix
  // size of |prefix_size| above. Hashes are sorted in lexicographic order.
  bytes raw_hashes = 2;
}

// The Rice-Golomb encoded data. Used for sending compressed 4-byte hashes or
// compressed removal indices.
message RiceDeltaEncoding {
  // The offset of the first entry in the encoded data, or, if only a single
  // integer was encoded, that single integer's value.
  int64 first_value = 1;

  // The Golomb-Rice parameter which is a number between 2 and 28. This field
  // is missing (that is, zero) if num_entries is zero.
  int32 rice_parameter = 2;

  // The number of entries that are delta encoded in the encoded data. If only a
  // single integer was encoded, this will be zero and the single value will be
  // stored in first_value.
  int32 num_entries = 3;

  // The encoded deltas that are encoded using the Golomb-Rice coder.
  bytes encoded_data = 4;
}

// The metadata associated with a specific threat entry. The client is expected
// to know the metadata key/value pairs associated with each threat type.
message ThreatEntryMetadata {
  // A single metadata entry.
  message MetadataEntry {
    // The metadata entry key.
    bytes key = 1;

    // The metadata entry value.
    bytes value = 2;
  }

  // The metadata entries.
  repeated MetadataEntry entries = 1;
}

// Describes an individual threat list. A list is defined by three parameters:
// the type of threat posed, the type of platform targeted by the threat, and
// the type of entries in the list.
message ThreatListDescriptor {
  // The threat type posed by the list's entries.
  ThreatType threat_type = 1;

  // The platform type targeted by the list's entries.
  PlatformType platform_type = 2;

  // The entry types contained in the list.
  ThreatEntryType threat_entry_type = 3;
}

// A collection of lists available for download by the client.
message ListThreatListsResponse {
  // The lists available for download by the client.
  repeated ThreatListDescriptor threat_lists = 1;
}