Merge pull request #1 from hannahrogers-google/seek-to-head

feat: adding ability to create subscriptions at head

.flake8

@@ -26,6 +26,7 @@ exclude =
   *_pb2.py
 
   # Standard linting exemptions.
+  **/.nox/**
   __pycache__,
   .git,
   *.pyc,

.github/header-checker-lint.yml

@@ -0,0 +1,15 @@
+{"allowedCopyrightHolders": ["Google LLC"],
+ "allowedLicenses": ["Apache-2.0", "MIT", "BSD-3"],
+ "ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt"],
+ "sourceFileExtensions": [
+ 	"ts", 
+ 	"js", 
+ 	"java", 
+ 	"sh", 
+ 	"Dockerfile", 
+ 	"yaml", 
+ 	"py",
+ 	"html",
+ 	"txt"
+ ]
+}

.github/sync-repo-settings.yaml

@@ -30,6 +30,10 @@ branchProtectionRules:
   requiredStatusCheckContexts:
     - 'Kokoro'
     - 'cla/google'
+    - 'Samples - Python 3.6'
+    - 'Samples - Python 3.7'
+    - 'Samples - Python 3.8'
+    - 'Samples - Lint'
 # List of explicit permissions to add (additive only)
 permissionRules:
     # Team slug to add to repository permissions

.gitignore

@@ -50,9 +50,10 @@ docs.metadata
 
 # Virtual environment
 env/
-venv/
+
+# Test logs
 coverage.xml
-sponge_log.xml
+*sponge_log.xml
 
 # System test environment variables.
 system_tests/local_test_setup

.kokoro/build.sh

@@ -15,7 +15,11 @@
 
 set -eo pipefail
 
-cd github/python-pubsublite
+if [[ -z "${PROJECT_ROOT:-}" ]]; then
+    PROJECT_ROOT="github/python-pubsublite"
+fi
+
+cd "${PROJECT_ROOT}"
 
 # Disable buffering, so that the logs stream through.
 export PYTHONUNBUFFERED=1
@@ -30,16 +34,26 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
 export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
 
 # Remove old nox
-python3.6 -m pip uninstall --yes --quiet nox-automation
+python3 -m pip uninstall --yes --quiet nox-automation
 
 # Install nox
-python3.6 -m pip install --upgrade --quiet nox
-python3.6 -m nox --version
+python3 -m pip install --upgrade --quiet nox
+python3 -m nox --version
+
+# If this is a continuous build, send the test log to the FlakyBot.
+# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot.
+if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then
+  cleanup() {
+    chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot
+    $KOKORO_GFILE_DIR/linux_amd64/flakybot
+  }
+  trap cleanup EXIT HUP
+fi
 
 # If NOX_SESSION is set, it only runs the specified session,
 # otherwise run all the sessions.
 if [[ -n "${NOX_SESSION:-}" ]]; then
-    python3.6 -m nox -s "${NOX_SESSION:-}"
+    python3 -m nox -s ${NOX_SESSION:-}
 else
-    python3.6 -m nox
+    python3 -m nox
 fi

.kokoro/docs/docs-presubmit.cfg

@@ -15,3 +15,14 @@ env_vars: {
     key: "TRAMPOLINE_IMAGE_UPLOAD"
     value: "false"
 }
+
+env_vars: {
+    key: "TRAMPOLINE_BUILD_FILE"
+    value: "github/python-pubsublite/.kokoro/build.sh"
+}
+
+# Only run this nox session.
+env_vars: {
+    key: "NOX_SESSION"
+    value: "docs docfx"
+}

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.4.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+-   repo: https://github.com/psf/black
+    rev: 19.10b0
+    hooks:
+    - id: black
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.8.4
+    hooks:
+    - id: flake8

.trampolinerc

@@ -24,6 +24,7 @@ required_envvars+=(
 pass_down_envvars+=(
     "STAGING_BUCKET"
     "V2_STAGING_BUCKET"
+    "NOX_SESSION"
 )
 
 # Prevent unintentional override on the default image.

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [0.3.0](https://www.github.com/googleapis/python-pubsublite/compare/v0.2.0...v0.3.0) (2021-03-09)
+
+
+### Features
+
+* Make message_id encode a PublishMetadata which includes the partition ([#90](https://www.github.com/googleapis/python-pubsublite/issues/90)) ([85944e7](https://www.github.com/googleapis/python-pubsublite/commit/85944e786908d0dd240b6c099cfd969045eecbd3))
+
+
+### Bug Fixes
+
+* remove absl dependency ([#94](https://www.github.com/googleapis/python-pubsublite/issues/94)) ([0573edb](https://www.github.com/googleapis/python-pubsublite/commit/0573edbefdf2612b2006b51829d1fd8fa636de3c))
+* Rename PublishMetadata to MessageMetadata ([#92](https://www.github.com/googleapis/python-pubsublite/issues/92)) ([a744441](https://www.github.com/googleapis/python-pubsublite/commit/a7444418d1e2822bcaee0da3aa036c6a14cf8a6e))
+
+
+### Documentation
+
+* remove multiprocessing note ([#82](https://www.github.com/googleapis/python-pubsublite/issues/82)) ([a8d26a1](https://www.github.com/googleapis/python-pubsublite/commit/a8d26a11db301d7dc7a97ff7f7405d82bcf0a910))
+
 ## [0.2.0](https://www.github.com/googleapis/python-pubsublite/compare/v0.1.0...v0.2.0) (2020-12-14)
 
 

CONTRIBUTING.rst

@@ -21,8 +21,8 @@ In order to add a feature:
 - The feature must be documented in both the API and narrative
   documentation.
 
-- The feature must work fully on the following CPython versions:  2.7,
-  3.5, 3.6, 3.7 and 3.8 on both UNIX and Windows.
+- The feature must work fully on the following CPython versions:
+  3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows.
 
 - The feature must not add unnecessary dependencies (where
   "unnecessary" is of course subjective, but new dependencies should
@@ -70,9 +70,14 @@ We use `nox <https://nox.readthedocs.io/en/latest/>`__ to instrument our tests.
 - To test your changes, run unit tests with ``nox``::
 
     $ nox -s unit-2.7
-    $ nox -s unit-3.7
+    $ nox -s unit-3.8
     $ ...
 
+- Args to pytest can be passed through the nox command separated by a `--`. For
+  example, to run a single test::
+
+    $ nox -s unit-3.8 -- -k <name of test>
+
   .. note::
 
     The unit tests and system tests are described in the
@@ -93,8 +98,12 @@ On Debian/Ubuntu::
 ************
 Coding Style
 ************
+- We use the automatic code formatter ``black``. You can run it using
+  the nox session ``blacken``. This will eliminate many lint errors. Run via::
 
-- PEP8 compliance, with exceptions defined in the linter configuration.
+   $ nox -s blacken
+
+- PEP8 compliance is required, with exceptions defined in the linter configuration.
   If you have ``nox`` installed, you can test that you have not introduced
   any non-compliant code via::
 
@@ -111,6 +120,16 @@ Coding Style
   should point to the official ``googleapis`` checkout and the
   the branch should be the main branch on that remote (``master``).
 
+- This repository contains configuration for the
+  `pre-commit <https://pre-commit.com/>`__ tool, which automates checking
+  our linters during a commit.  If you have it installed on your ``$PATH``,
+  you can enable enforcing those checks via:
+
+.. code-block:: bash
+
+   $ pre-commit install
+   pre-commit installed at .git/hooks/pre-commit
+
 Exceptions to PEP8:
 
 - Many unit tests use a helper method, ``_call_fut`` ("FUT" is short for
@@ -123,13 +142,18 @@ Running System Tests
 
 - To run system tests, you can execute::
 
-   $ nox -s system-3.7
+   # Run all system tests
+   $ nox -s system-3.8
    $ nox -s system-2.7
 
+   # Run a single system test
+   $ nox -s system-3.8 -- -k <name of test>
+
+
   .. note::
 
       System tests are only configured to run under Python 2.7 and
-      Python 3.7. For expediency, we do not run them in older versions
+      Python 3.8. For expediency, we do not run them in older versions
       of Python 3.
 
   This alone will not run the tests. You'll need to change some local
@@ -192,25 +216,24 @@ Supported Python Versions
 
 We support:
 
--  `Python 3.5`_
 -  `Python 3.6`_
 -  `Python 3.7`_
 -  `Python 3.8`_
+-  `Python 3.9`_
 
-.. _Python 3.5: https://docs.python.org/3.5/
 .. _Python 3.6: https://docs.python.org/3.6/
 .. _Python 3.7: https://docs.python.org/3.7/
 .. _Python 3.8: https://docs.python.org/3.8/
+.. _Python 3.9: https://docs.python.org/3.9/
 
 
 Supported versions can be found in our ``noxfile.py`` `config`_.
 
 .. _config: https://github.com/googleapis/python-pubsublite/blob/master/noxfile.py
 
-Python 2.7 support is deprecated. All code changes should maintain Python 2.7 compatibility until January 1, 2020.
 
 We also explicitly decided to support Python 3 beginning with version
-3.5. Reasons for this include:
+3.6. Reasons for this include:
 
 -  Encouraging use of newest versions of Python 3
 -  Taking the lead of `prominent`_ open-source `projects`_

LICENSE

@@ -1,6 +1,7 @@
-                            Apache License
+
+                                 Apache License
                            Version 2.0, January 2004
-                        https://www.apache.org/licenses/
+                        http://www.apache.org/licenses/
 
    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
@@ -192,7 +193,7 @@
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at
 
-       https://www.apache.org/licenses/LICENSE-2.0
+       http://www.apache.org/licenses/LICENSE-2.0
 
    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,

MANIFEST.in

@@ -16,10 +16,10 @@
 
 # Generated by synthtool. DO NOT EDIT!
 include README.rst LICENSE
-recursive-include google *.json *.proto
+recursive-include google *.json *.proto py.typed
 recursive-include tests *
 global-exclude *.py[co]
 global-exclude __pycache__
 
 # Exclude scripts for samples readmegen
-prune scripts/readme-gen
+prune scripts/readme-gen

docs/_static/custom.css

@@ -1,4 +1,9 @@
 div#python2-eol {
 	border-color: red;
 	border-width: medium;
-} 
+} 
+
+/* Ensure minimum width for 'Parameters' / 'Returns' column */
+dl.field-list > dt {
+    min-width: 100px
+}

docs/conf.py

@@ -345,8 +345,8 @@
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
-    "python": ("https://python.readthedocs.io/en/latest/", None),
-    "google-auth": ("https://google-auth.readthedocs.io/en/stable", None),
+    "python": ("https://python.readthedocs.org/en/latest/", None),
+    "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None),
     "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,),
     "grpc": ("https://grpc.github.io/grpc/python/", None),
     "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None),

google/cloud/pubsublite/admin_client.py

@@ -30,6 +30,7 @@
     SubscriptionPath,
     LocationPath,
     TopicPath,
+    OffsetLocation,
 )
 from google.cloud.pubsublite_v1 import AdminServiceClient, Subscription, Topic
 
@@ -104,6 +105,12 @@ def list_topic_subscriptions(self, topic_path: TopicPath):
     def create_subscription(self, subscription: Subscription) -> Subscription:
         return self._impl.create_subscription(subscription)
 
+    @overrides
+    def create_subscription_at_offset(
+        self, subscription: Subscription, starting_offset: OffsetLocation
+    ) -> Subscription:
+        return self._impl.create_subscription(subscription, starting_offset)
+
     @overrides
     def get_subscription(self, subscription_path: SubscriptionPath) -> Subscription:
         return self._impl.get_subscription(subscription_path)

google/cloud/pubsublite/admin_client_interface.py

@@ -20,6 +20,7 @@
     TopicPath,
     LocationPath,
     SubscriptionPath,
+    OffsetLocation,
 )
 from google.cloud.pubsublite_v1 import Topic, Subscription
 from google.protobuf.field_mask_pb2 import FieldMask
@@ -66,6 +67,12 @@ def list_topic_subscriptions(self, topic_path: TopicPath):
     def create_subscription(self, subscription: Subscription) -> Subscription:
         """Create a subscription, returns the created subscription."""
 
+    @abstractmethod
+    def create_subscription_at_offset(
+        self, subscription: Subscription, starting_offset: OffsetLocation
+    ) -> Subscription:
+        """Create a subscription at the given starting offset, returns the created subscription."""
+
     @abstractmethod
     def get_subscription(self, subscription_path: SubscriptionPath) -> Subscription:
         """Get the subscription object from the server."""

google/cloud/pubsublite/internal/wire/admin_client_impl.py

@@ -22,6 +22,7 @@
     SubscriptionPath,
     LocationPath,
     TopicPath,
+    OffsetLocation,
 )
 from google.cloud.pubsublite_v1 import (
     Subscription,
@@ -73,11 +74,17 @@ def list_topic_subscriptions(self, topic_path: TopicPath):
         return [SubscriptionPath.parse(x) for x in subscription_strings]
 
     def create_subscription(self, subscription: Subscription) -> Subscription:
+        return self.create_subscription_at_offset(subscription, OffsetLocation.END)
+
+    def create_subscription_at_offset(
+        self, subscription: Subscription, starting_offset: OffsetLocation
+    ) -> Subscription:
         path = SubscriptionPath.parse(subscription.name)
         return self._underlying.create_subscription(
             parent=str(path.to_location_path()),
             subscription=subscription,
             subscription_id=path.name,
+            skip_backlog=(starting_offset == OffsetLocation.END),
         )
 
     def get_subscription(self, subscription_path: SubscriptionPath) -> Subscription: