From 9165828a0fb9afde3e2d465b3b9d2e5208da660c Mon Sep 17 00:00:00 2001 From: David Wrede Date: Tue, 11 Feb 2020 16:04:28 -0800 Subject: [PATCH] Remove tables and refer to dependency locations in code (#10745) --- .../documentation/sdks/java-dependencies.md | 355 ++----------- .../documentation/sdks/python-dependencies.md | 473 +----------------- 2 files changed, 62 insertions(+), 766 deletions(-) diff --git a/website/src/documentation/sdks/java-dependencies.md b/website/src/documentation/sdks/java-dependencies.md index 451e2529414a8..d6142088fb1bb 100644 --- a/website/src/documentation/sdks/java-dependencies.md +++ b/website/src/documentation/sdks/java-dependencies.md @@ -26,336 +26,51 @@ behavior in the service. If you are using any of these packages in your code, be aware that some libraries are not forward-compatible and you may need to pin to the listed versions that will be in scope during execution. -

To see the compile and runtime dependencies for your Beam SDK version, expand -the relevant section below.

+Compile and runtime dependencies for your Beam SDK version are listed in `BeamModulePlugin.groovy` in the Beam repository. To view them, perform the following steps: -
2.9.0 +1. Open `BeamModulePlugin.groovy`. -

Beam SDK for Java 2.9.0 has the following compile and runtime dependencies.

+ ``` + https://raw.githubusercontent.com/apache/beam/v/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy + ``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
GroupIdArtifactIdVersion
org.apache.activemqactivemq-amqp5.13.1
org.apache.activemqactivemq-broker5.13.1
org.apache.activemqactivemq-client5.13.1
org.apache.activemqactivemq-jaas5.13.1
org.apache.activemq.toolingactivemq-junit5.13.1
org.apache.activemqactivemq-kahadb-store5.13.1
org.apache.activemqactivemq-mqtt5.13.1
org.apache.apexapex-common3.7.0
org.apache.apexapex-engine3.7.0
args4jargs4j2.33
org.apache.avroavro1.8.2
com.google.cloud.bigdataossgcsio1.9.0
com.google.cloud.bigdataossutil1.9.0
com.google.cloud.bigtablebigtable-client-core1.4.0
net.bytebuddybyte-buddy1.9.3
org.apache.commonscommons-compress1.16.1
org.apache.commonscommons-csv1.4
commons-iocommons-io1.3.2
commons-iocommons-io2.5
org.apache.commonscommons-lang33.6
org.apache.commonscommons-math33.6.1
com.google.cloud.datastoredatastore-v1-proto-client1.6.0
com.google.errorproneerror_prone_annotations2.0.15
com.google.apigax-grpc1.29.0
com.google.api-clientgoogle-api-client1.27.0
com.google.api-clientgoogle-api-client-jackson21.27.0
com.google.api-clientgoogle-api-client-java61.27.0
com.google.apisgoogle-api-services-bigqueryv2-rev20181104-1.27.0
com.google.apisgoogle-api-services-clouddebuggerv2-rev20180801-1.27.0
com.google.apisgoogle-api-services-cloudresourcemanagerv1-rev20181015-1.27.0
com.google.apisgoogle-api-services-dataflowv1b3-rev20181107-1.27.0
com.google.apisgoogle-api-services-pubsubv1-rev20181105-1.27.0
com.google.apisgoogle-api-services-storagev1-rev20181013-1.27.0
com.google.authgoogle-auth-library-credentials0.10.0
com.google.authgoogle-auth-library-oauth2-http0.10.0
com.google.cloudgoogle-cloud-bigquery1.27.0
com.google.cloudgoogle-cloud-core1.36.0
com.google.cloudgoogle-cloud-core-grpc1.36.0
com.google.cloud.dataflowgoogle-cloud-dataflow-java-proto-library-all0.5.160304
com.google.cloudgoogle-cloud-spanner0.54.0-beta
com.google.http-clientgoogle-http-client1.27.0
com.google.http-clientgoogle-http-client-jackson1.27.0
com.google.http-clientgoogle-http-client-jackson21.27.0
com.google.http-clientgoogle-http-client-protobuf1.27.0
com.google.oauth-clientgoogle-oauth-client1.27.0
com.google.oauth-clientgoogle-oauth-client-java61.27.0
io.grpcgrpc-all1.13.1
io.grpcgrpc-auth1.13.1
io.grpcgrpc-core1.13.1
com.google.api.grpcgrpc-google-cloud-pubsub-v11.18.0
io.grpcgrpc-netty1.13.1
io.grpcgrpc-protobuf-lite1.13.1
io.grpcgrpc-protobuf1.13.1
io.grpcgrpc-stub1.13.1
com.google.guavaguava20.0
com.google.guavaguava-testlib20.0
org.apache.hadoophadoop-client2.7.3
org.apache.hadoophadoop-common2.7.3
org.apache.hadoophadoop-hdfs2.7.3
org.apache.hadoophadoop-mapreduce-client-core2.7.3
org.apache.hadoophadoop-minicluster2.7.3
org.hamcresthamcrest-core1.3
org.hamcresthamcrest-library1.3
com.fasterxml.jackson.corejackson-annotations2.9.5
com.fasterxml.jackson.corejackson-core2.9.5
com.fasterxml.jackson.corejackson-databind2.9.5
com.fasterxml.jackson.dataformatjackson-dataformat-cbor2.9.5
com.fasterxml.jackson.dataformatjackson-dataformat-yaml2.9.5
com.fasterxml.jackson.datatypejackson-datatype-joda2.9.5
com.fasterxml.jackson.modulejackson-module-scala_2.112.9.5
javax.xml.bindjaxb-api2.2.12
joda-timejoda-time2.4
junitjunit4.12
org.apache.kafkakafka_2.111.0.0
org.apache.kafkakafka-clients1.0.0
org.apache.apexmalhar-library3.4.0
org.mockitomockito-core1.10.19
io.nettynetty-handler4.1.25.Final
io.nettynetty-tcnative-boringssl-static2.0.8.Final
io.nettynetty-transport-native-epoll4.1.25.Final
org.postgresqlpostgresql42.2.2
org.powermockpowermock-mockito-release-full1.6.4
com.google.protobufprotobuf-java3.6.0
com.google.protobufprotobuf-java-util3.6.0
com.google.api.grpcproto-google-cloud-pubsub-v11.18.0
com.google.api.grpcproto-google-cloud-spanner-admin-database-v10.19.0
com.google.api.grpcproto-google-common-protos1.12.0
org.slf4jslf4j-api1.7.25
org.slf4jslf4j-jdk141.7.25
org.slf4jslf4j-log4j121.7.25
org.slf4jslf4j-simple1.7.25
org.xerial.snappysnappy-java1.1.4
org.apache.sparkspark-core_2.112.3.2
org.apache.sparkspark-network-common_2.112.3.2
org.apache.sparkspark-streaming_2.112.3.2
org.codehaus.woodstoxstax2-api3.1.4
org.codehaus.woodstoxwoodstox-core-asl4.4.1
com.pholserjunit-quickcheck-core0.8
+ Replace `` with the major.minor.patch version of the SDK. For example, {:target="_blank"} will provide the dependencies for the {{ site.release_latest }} release. + +2. Review the list under `project.ext.library`. -
+ **Note:** Some dependencies in the list use version variables, such as `google_auth_version`. These variables are defined prior to the `project.ext.library` map definition. -
2.8.0 +You can also retrieve this list by creating a new project through Maven and resolving the dependencies. -

Beam SDK for Java 2.8.0 has the following compile and runtime dependencies.

+1. Define the Beam SDK and Java versions for the new project. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
GroupIdArtifactIdVersion
org.apache.activemqactivemq-amqp5.13.1
org.apache.activemqactivemq-broker5.13.1
org.apache.activemqactivemq-client5.13.1
org.apache.activemqactivemq-jaas5.13.1
org.apache.activemq.toolingactivemq-junit5.13.1
org.apache.activemqactivemq-kahadb-store5.13.1
org.apache.activemqactivemq-mqtt5.13.1
org.apache.apexapex-common3.7.0
org.apache.apexapex-engine3.7.0
com.google.apiapi-common1.6.0
args4jargs4j2.33
org.apache.avroavro1.8.2
com.google.cloud.bigtablebigtable-client-core1.4.0
net.bytebuddybyte-buddy1.8.11
org.apache.commonscommons-compress1.16.1
org.apache.commonscommons-csv1.4
commons-iocommons-io1.3.2
commons-iocommons-io2.5
org.apache.commonscommons-lang33.6
org.apache.commonscommons-math33.6.1
com.google.cloud.datastoredatastore-v1-proto-client1.6.0
com.google.errorproneerror_prone_annotations2.0.15
com.google.apigax-grpc1.29.0
com.google.cloud.bigdataossgcsio1.9.0
com.google.api-clientgoogle-api-client-jackson21.24.1
com.google.api-clientgoogle-api-client-java61.24.1
com.google.api-clientgoogle-api-client1.24.1
com.google.apisgoogle-api-services-bigqueryv2-rev402-1.24.1
com.google.apisgoogle-api-services-clouddebuggerv2-rev253-1.24.1
com.google.apisgoogle-api-services-cloudresourcemanagerv1-rev502-1.24.1
com.google.apisgoogle-api-services-dataflowv1b3-rev257-1.24.1
com.google.apisgoogle-api-services-pubsubv1-rev399-1.24.1
com.google.apisgoogle-api-services-storagev1-rev136-1.24.1
com.google.authgoogle-auth-library-credentials0.10.0
com.google.authgoogle-auth-library-oauth2-http0.10.0
com.google.cloudgoogle-cloud-core-grpc1.36.0
com.google.cloudgoogle-cloud-core1.36.0
com.google.cloud.dataflowgoogle-cloud-dataflow-java-proto-library-all0.5.160304
com.google.cloudgoogle-cloud-spanner0.54.0-beta
com.google.http-clientgoogle-http-client-jackson21.24.1
com.google.http-clientgoogle-http-client-jackson1.24.1
com.google.http-clientgoogle-http-client-protobuf1.24.1
com.google.http-clientgoogle-http-client1.24.1
com.google.oauth-clientgoogle-oauth-client-java61.24.1
com.google.oauth-clientgoogle-oauth-client1.24.1
io.grpcgrpc-all1.13.1
io.grpcgrpc-auth1.13.1
io.grpcgrpc-core1.13.1
com.google.api.grpcgrpc-google-cloud-bigtable-v20.19.0
com.google.api.grpcgrpc-google-cloud-pubsub-v11.18.0
io.grpcgrpc-netty1.13.1
io.grpcgrpc-protobuf-lite1.13.1
io.grpcgrpc-protobuf1.13.1
io.grpcgrpc-stub1.13.1
com.google.guavaguava20.0
com.google.guavaguava-testlib20.0
org.apache.hadoophadoop-client2.7.3
org.apache.hadoophadoop-common2.7.3
org.apache.hadoophadoop-hdfs2.7.3
org.apache.hadoophadoop-hdfs2.7.3
org.apache.hadoophadoop-mapreduce-client-core2.7.3
org.apache.hadoophadoop-minicluster2.7.3
org.hamcresthamcrest-core1.3
org.hamcresthamcrest-library1.3
com.fasterxml.jackson.corejackson-annotations2.9.5
com.fasterxml.jackson.corejackson-core2.9.5
com.fasterxml.jackson.corejackson-databind2.9.5
com.fasterxml.jackson.dataformatjackson-dataformat-cbor2.9.5
com.fasterxml.jackson.dataformatjackson-dataformat-yaml2.9.5
com.fasterxml.jackson.datatypejackson-datatype-joda2.9.5
com.fasterxml.jackson.modulejackson-module-scala_2.112.9.5
javax.xml.bindjaxb-api2.2.12
joda-timejoda-time2.4
com.pholserjunit-quickcheck-core0.8
junitjunit4.12
org.apache.kafkakafka_2.111.0.0
org.apache.kafkakafka-clients1.0.0
org.apache.apexmalhar-library3.4.0
org.mockitomockito-core1.10.19
io.nettynetty-handler4.1.25.Final
io.nettynetty-tcnative-boringssl-static2.0.8.Final
io.nettynetty-transport-native-epoll4.1.25.Final
org.postgresqlpostgresql42.2.2
org.powermockpowermock-mockito-release-full1.6.4
com.google.protobufprotobuf-java3.6.0
com.google.protobufprotobuf-java-util3.6.0
com.google.api.grpcproto-google-cloud-datastore-v10.19.0
com.google.api.grpcproto-google-cloud-pubsub-v11.18.0
com.google.api.grpcproto-google-cloud-spanner-admin-database-v10.19.0
com.google.api.grpcproto-google-common-protos1.12.0
org.slf4jslf4j-api1.7.25
org.slf4jslf4j-jdk141.7.25
org.slf4jslf4j-log4j121.7.25
org.slf4jslf4j-simple1.7.25
org.xerial.snappysnappy-java1.1.4
org.apache.sparkspark-core_2.112.3.1
org.apache.sparkspark-network-common_2.112.3.1
org.apache.sparkspark-streaming_2.112.3.1
org.codehaus.woodstoxstax2-api3.1.4
com.google.cloud.bigdataossutil1.9.0
org.codehaus.woodstoxwoodstox-core-asl4.4.1
+ ``` + export BEAM_VERSION={{ site.release_latest }} + export JAVA_VERSION=11 + ``` -
+2. Create the project. -
2.7.0 + ``` + mvn archetype:generate \ + -DinteractiveMode=false \ + -DarchetypeGroupId=org.apache.beam \ + -DarchetypeArtifactId=beam-sdks-java-maven-archetypes-starter \ + -DarchetypeVersion=$BEAM_VERSION \ + -DtargetPlatform=$JAVA_VERSION \ + -DartifactId=check-pipeline-dependencies \ + -DgroupId=org.apache.beam.samples + ``` -

Beam SDK for Java 2.7.0 has the following compile and runtime dependencies.

+3. Change to the new project directory. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
GroupIdArtifactIdVersion
org.apache.activemqactivemq-amqp5.13.1
org.apache.activemqactivemq-broker5.13.1
org.apache.activemqactivemq-client5.13.1
org.apache.activemqactivemq-jaas5.13.1
org.apache.activemq.toolingactivemq-junit5.13.1
org.apache.activemqactivemq-kahadb-store5.13.1
org.apache.activemqactivemq-mqtt5.13.1
org.apache.apexapex-common3.7.0
org.apache.apexapex-engine3.7.0
com.google.apiapi-common1.6.0
args4jargs4j2.33
org.apache.avroavro1.8.2
com.google.cloud.bigtablebigtable-client-core1.4.0
net.bytebuddybyte-buddy1.8.11
org.apache.commonscommons-compress1.16.1
org.apache.commonscommons-csv1.4
commons-iocommons-io1.3.2
commons-iocommons-io2.5
org.apache.commonscommons-lang33.6
org.apache.commonscommons-math33.6.1
com.google.cloud.datastoredatastore-v1-proto-client1.6.0
com.google.errorproneerror_prone_annotations2.0.15
com.google.apigax-grpc1.29.0
com.google.cloud.bigdataossgcsio1.9.0
com.google.api-clientgoogle-api-client-jackson21.23.0
com.google.api-clientgoogle-api-client-java61.23.0
com.google.api-clientgoogle-api-client1.23.0
com.google.apisgoogle-api-services-bigqueryv2-rev374-1.23.0
com.google.apisgoogle-api-services-clouddebuggerv2-rev233-1.23.0
com.google.apisgoogle-api-services-cloudresourcemanagerv1-rev477-1.23.0
com.google.apisgoogle-api-services-dataflowv1b3-rev221-1.23.0
com.google.apisgoogle-api-services-pubsubv1-rev382-1.23.0
com.google.apisgoogle-api-services-storagev1-rev124-1.23.0
com.google.authgoogle-auth-library-credentials0.10.0
com.google.authgoogle-auth-library-oauth2-http0.10.0
com.google.cloudgoogle-cloud-core-grpc1.36.0
com.google.cloudgoogle-cloud-core1.36.0
com.google.cloud.dataflowgoogle-cloud-dataflow-java-proto-library-all0.5.160304
com.google.cloudgoogle-cloud-spanner0.54.0-beta
com.google.http-clientgoogle-http-client-jackson21.23.0
com.google.http-clientgoogle-http-client-jackson1.23.0
com.google.http-clientgoogle-http-client-protobuf1.23.0
com.google.http-clientgoogle-http-client1.23.0
com.google.oauth-clientgoogle-oauth-client-java61.23.0
com.google.oauth-clientgoogle-oauth-client1.23.0
io.grpcgrpc-all1.13.1
io.grpcgrpc-auth1.13.1
io.grpcgrpc-core1.13.1
com.google.api.grpcgrpc-google-cloud-bigtable-v20.19.0
com.google.api.grpcgrpc-google-cloud-pubsub-v11.18.0
io.grpcgrpc-netty1.13.1
io.grpcgrpc-protobuf-lite1.13.1
io.grpcgrpc-protobuf1.13.1
io.grpcgrpc-stub1.13.1
com.google.guavaguava20.0
com.google.guavaguava-testlib20.0
org.apache.hadoophadoop-client2.7.3
org.apache.hadoophadoop-common2.7.3
org.apache.hadoophadoop-hdfs2.7.3
org.apache.hadoophadoop-hdfs2.7.3
org.apache.hadoophadoop-mapreduce-client-core2.7.3
org.apache.hadoophadoop-minicluster2.7.3
org.hamcresthamcrest-core1.3
org.hamcresthamcrest-library1.3
com.fasterxml.jackson.corejackson-annotations2.9.5
com.fasterxml.jackson.corejackson-core2.9.5
com.fasterxml.jackson.corejackson-databind2.9.5
com.fasterxml.jackson.dataformatjackson-dataformat-cbor2.9.5
com.fasterxml.jackson.dataformatjackson-dataformat-yaml2.9.5
com.fasterxml.jackson.datatypejackson-datatype-joda2.9.5
com.fasterxml.jackson.modulejackson-module-scala_2.112.9.5
javax.xml.bindjaxb-api2.2.12
joda-timejoda-time2.4
com.pholserjunit-quickcheck-core0.8
junitjunit4.12
org.apache.kafkakafka_2.111.0.0
org.apache.kafkakafka-clients1.0.0
org.apache.apexmalhar-library3.4.0
org.mockitomockito-core1.10.19
io.nettynetty-handler4.1.25.Final
io.nettynetty-tcnative-boringssl-static2.0.8.Final
io.nettynetty-transport-native-epoll4.1.25.Final
org.postgresqlpostgresql42.2.2
org.powermockpowermock-mockito-release-full1.6.4
com.google.protobufprotobuf-java3.6.0
com.google.protobufprotobuf-java-util3.6.0
com.google.api.grpcproto-google-cloud-datastore-v10.19.0
com.google.api.grpcproto-google-cloud-pubsub-v11.18.0
com.google.api.grpcproto-google-cloud-spanner-admin-database-v10.19.0
com.google.api.grpcproto-google-common-protos1.12.0
org.slf4jslf4j-api1.7.25
org.slf4jslf4j-jdk141.7.25
org.slf4jslf4j-log4j121.7.25
org.slf4jslf4j-simple1.7.25
org.xerial.snappysnappy-java1.1.4
org.apache.sparkspark-core_2.112.3.1
org.apache.sparkspark-network-common_2.112.3.1
org.apache.sparkspark-streaming_2.112.3.1
org.codehaus.woodstoxstax2-api3.1.4
com.google.cloud.bigdataossutil1.9.0
org.codehaus.woodstoxwoodstox-core-asl4.4.1
+ ``` + cd check-pipeline-dependencies + ``` -
+4. Resolve and list the dependencies. + + ``` + mvn dependency:resolve && mvn -o dependency:list + ``` diff --git a/website/src/documentation/sdks/python-dependencies.md b/website/src/documentation/sdks/python-dependencies.md index a983bd64d77e2..6f5cb4c66c238 100644 --- a/website/src/documentation/sdks/python-dependencies.md +++ b/website/src/documentation/sdks/python-dependencies.md @@ -26,460 +26,41 @@ behavior in the service. If you are using any of these packages in your code, be aware that some libraries are not forward-compatible and you may need to pin to the listed versions that will be in scope during execution. -

To see the compile and runtime dependencies for your Beam SDK version, expand -the relevant section below.

+Dependencies for your Beam SDK version are listed in `setup.py` in the Beam repository. To view them, perform the following steps: -
2.17.0 +1. Open `setup.py`. -

Beam SDK for Python 2.17.0 has the following compile and runtime dependencies.

+ ``` + https://raw.githubusercontent.com/apache/beam/v/sdks/python/setup.py + ``` + + Replace `` with the major.minor.patch version of the SDK. For example, {:target="_blank"} will provide the dependencies for the {{ site.release_latest }} release. + + +2. Review the core dependency list under `REQUIRED_PACKAGES`. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro-python3>=1.8.1,<2.0.0; python_version >= "3.0"
avro>=1.8.1,<2.0.0; python_version < "3.0"
cachetools>=3.1.0,<4
crcmod>=1.7,<2.0
dill>=0.3.0,<0.3.1
fastavro>=0.21.4,<0.22
funcsigs>=1.0.2,<2; python_version < "3.0"
future>=0.16.0,<1.0.0
futures>=3.2.0,<4.0.0; python_version < "3.0"
google-apitools>=0.5.28,<0.5.29
google-cloud-bigquery>=1.6.0,<1.18.0
google-cloud-bigtable>=0.31.1,<1.1.0
google-cloud-core>=0.28.1,<2
google-cloud-datastore>=1.7.1,<1.8.0
google-cloud-pubsub>=0.39.0,<1.1.0
googledatastore>=7.0.1,<7.1; python_version < "3.0"
grpcio>=1.12.1,<2
hdfs>=2.1.0,<3.0.0
httplib2>=0.8,<=0.12.0
mock>=1.0.1,<3.0.0
oauth2client>=2.0.1,<4
proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4; python_version < "3.0"
protobuf>=3.5.0.post1,<4
pyarrow>=0.15.1,<0.16.0; python_version >= "3.0" or platform_system != "Windows"
pydot>=1.2.0,<2
pymongo>=3.8.0,<4.0.0
python-dateutil>=2.8.0,<3
pytz>=2018.3
pyvcf>=0.6.8,<0.7.0; python_version < "3.0"
typing>=3.6.0,<3.7.0; python_version < "3.5.0"
+ **Note:** If you require [extra features]({{ site.baseurl }}/get-started/quickstart-py#extra-requirements) such as `gcp` or `test`, you should review the lists under `REQUIRED_TEST_PACKAGES`, `GCP_REQUIREMENTS`, or `INTERACTIVE_BEAM` for additional dependencies. -
+You can also retrieve the dependency list from the command line using the following process: -
2.16.0 +1. Create a clean virtual environment on your local machine. -

Beam SDK for Python 2.16.0 has the following compile and - runtime dependencies.

+ Python 3: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro-python3>=1.8.1,<2.0.0; python_version >= "3.0"
avro>=1.8.1,<2.0.0; python_version < "3.0"
cachetools>=3.1.0,<4
crcmod>=1.7,<2.0
dill>=0.3.0,<0.3.1
fastavro>=0.21.4,<0.22
funcsigs>=1.0.2,<2; python_version < "3.0"
future>=0.16.0,<1.0.0
futures>=3.2.0,<4.0.0; python_version < "3.0"
google-apitools>=0.5.28,<0.5.29
google-cloud-bigquery>=1.6.0,<1.18.0
google-cloud-bigtable>=0.31.1,<1.1.0
google-cloud-core>=0.28.1,<2
google-cloud-datastore>=1.7.1,<1.8.0
google-cloud-pubsub>=0.39.0,<1.1.0
googledatastore>=7.0.1,<7.1; python_version < "3.0"
grpcio>=1.12.1,<2
hdfs>=2.1.0,<3.0.0
httplib2>=0.8,<=0.12.0
mock>=1.0.1,<3.0.0
oauth2client>=2.0.1,<4
proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4; python_version < "3.0"
protobuf>=3.5.0.post1,<4
pyarrow>=0.11.1,<0.15.0; python_version >= "3.0" or platform_system != "Windows"
pydot>=1.2.0,<2
pymongo>=3.8.0,<4.0.0
python-dateutil>=2.8.0,<3
pytz>=2018.3
pyvcf>=0.6.8,<0.7.0; python_version < "3.0"
pyyaml>=3.12,<4.0.0
typing>=3.6.0,<3.7.0; python_version < "3.5.0"
+ ``` + $ python3 -m venv env && source env/bin/activate + ``` + + Python 2: -
+ ``` + $ pip install virtualenv && virtualenv env && source env/bin/activate + ``` -
2.15.0 +2. [Install the Beam Python SDK]({{ site.baseurl }}/get-started/quickstart-py/#download-and-install). -

Beam SDK for Python 2.15.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro-python3>=1.8.1,<2.0.0; python_version >= "3.0"
avro>=1.8.1,<2.0.0; python_version < "3.0"
cachetools>=3.1.0,<4
crcmod>=1.7,<2.0
dill>=0.2.9,<0.2.10
fastavro>=0.21.4,<0.22
future>=0.16.0,<1.0.0
futures>=3.2.0,<4.0.0; python_version < "3.0"
google-apitools>=0.5.28,<0.5.29
google-cloud-bigquery>=1.6.0,<1.18.0
google-cloud-bigtable>=0.31.1,<0.33.0
google-cloud-core>=0.28.1,<2
google-cloud-datastore>=1.7.1,<1.8.0
google-cloud-pubsub>=0.39.0,<0.40.0
googledatastore>=7.0.1,<7.1; python_version < "3.0"
grpcio>=1.8,<2
hdfs>=2.1.0,<3.0.0
httplib2>=0.8,<=0.12.0
mock>=1.0.1,<3.0.0
oauth2client>=2.0.1,<4
proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4; python_version < "3.0"
protobuf>=3.5.0.post1,<4
pyarrow>=0.11.1,<0.15.0; python_version >= "3.0" or platform_system != "Windows"
pydot>=1.2.0,<2
pymongo>=3.8.0,<4.0.0
pytz>=2018.3
pyvcf>=0.6.8,<0.7.0; python_version < "3.0"
pyyaml>=3.12,<4.0.0
typing>=3.6.0,<3.7.0; python_version < "3.5.0"
+3. Retrieve the list of dependencies. -
- -
2.14.0 - -

Beam SDK for Python 2.14.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro-python3>=1.8.1,<2.0.0; python_version >= "3.0"
avro>=1.8.1,<2.0.0; python_version < "3.0"
cachetools>=3.1.0,<4
crcmod>=1.7,<2.0
dill>=0.2.9,<0.2.10
fastavro>=0.21.4,<0.22
future>=0.16.0,<1.0.0
futures>=3.2.0,<4.0.0; python_version < "3.0"
google-apitools>=0.5.28,<0.5.29
google-cloud-bigquery>=1.6.0,<1.7.0
google-cloud-bigtable>=0.31.1,<0.33.0
google-cloud-core>=0.28.1,<0.30.0
google-cloud-datastore>=1.7.1,<1.8.0
google-cloud-pubsub>=0.39.0,<0.40.0
googledatastore>=7.0.1,<7.1; python_version < "3.0"
grpcio>=1.8,<2
hdfs>=2.1.0,<3.0.0
httplib2>=0.8,<=0.12.0
mock>=1.0.1,<3.0.0
oauth2client>=2.0.1,<4
proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4; python_version < "3.0"
protobuf>=3.5.0.post1,<4
pyarrow>=0.11.1,<0.15.0; python_version >= "3.0" or platform_system != "Windows"
pydot>=1.2.0,<1.3
pymongo>=3.8.0,<4.0.0
pytz>=2018.3
pyvcf>=0.6.8,<0.7.0; python_version < "3.0"
pyyaml>=3.12,<4.0.0
typing>=3.6.0,<3.7.0; python_version < "3.5.0"
- -
- -
2.13.0 - -

Beam SDK for Python 2.13.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro-python3>=1.8.1,<2.0.0; python_version >= "3.0"
avro>=1.8.1,<2.0.0; python_version < "3.0"
cachetools>=3.1.0,<4
crcmod>=1.7,<2.0
dill>=0.2.9,<0.2.10
fastavro>=0.21.4,<0.22
future>=0.16.0,<1.0.0
futures>=3.2.0,<4.0.0; python_version < "3.0"
google-apitools>=0.5.28,<0.5.29
google-cloud-bigquery>=1.6.0,<1.7.0
google-cloud-bigtable>=0.31.1,<0.33.0
google-cloud-core>=0.28.1,<0.30.0
google-cloud-datastore>=1.7.1,<1.8.0
google-cloud-pubsub>=0.39.0,<0.40.0
googledatastore>=7.0.1,<7.1; python_version < "3.0"
grpcio>=1.8,<2
hdfs>=2.1.0,<3.0.0
httplib2>=0.8,<=0.12.0
mock>=1.0.1,<3.0.0
oauth2client>=2.0.1,<4
proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4; python_version < "3.0"
protobuf>=3.5.0.post1,<4
pyarrow>=0.11.1,<0.14.0; python_version >= "3.0" or platform_system != "Windows"
pydot>=1.2.0,<1.3
pytz>=2018.3
pyvcf>=0.6.8,<0.7.0; python_version < "3.0"
pyyaml>=3.12,<4.0.0
typing>=3.6.0,<3.7.0; python_version < "3.5.0"
- -
- -
2.12.0 - -

Beam SDK for Python 2.12.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro-python3>=1.8.1,<2.0.0; python_version >= "3.0"
avro>=1.8.1,<2.0.0; python_version < "3.0"
crcmod>=1.7,<2.0
dill>=0.2.9,<0.2.10
fastavro>=0.21.4,<0.22
future>=0.16.0,<1.0.0
futures>=3.2.0,<4.0.0; python_version < "3.0"
google-apitools>=0.5.26,<0.5.27
google-cloud-bigquery>=1.6.0,<1.7.0
google-cloud-bigtable==0.31.1
google-cloud-core==0.28.1
google-cloud-pubsub==0.39.0
googledatastore>=7.0.1,<7.1; python_version < "3.0"
grpcio>=1.8,<2
hdfs>=2.1.0,<3.0.0
httplib2>=0.8,<=0.11.3
mock>=1.0.1,<3.0.0
oauth2client>=2.0.1,<4
proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4
protobuf>=3.5.0.post1,<4
pyarrow>=0.11.1,<0.12.0; python_version >= "3.0" or platform_system != "Windows"
pydot>=1.2.0,<1.3
pytz>=2018.3
pyvcf>=0.6.8,<0.7.0; python_version < "3.0"
pyyaml>=3.12,<4.0.0
typing>=3.6.0,<3.7.0; python_version < "3.5.0"
- -
- -
2.11.0 - -

Beam SDK for Python 2.11.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro-python3>=1.8.1,<2.0.0; python_version >= "3.0"
avro>=1.8.1,<2.0.0; python_version < "3.0"
crcmod>=1.7,<2.0
dill>=0.2.9,<0.2.10
fastavro>=0.21.4,<0.22
future>=0.16.0,<1.0.0
futures>=3.2.0,<4.0.0; python_version < "3.0"
google-apitools>=0.5.26,<0.5.27
google-cloud-bigquery>=1.6.0,<1.7.0
google-cloud-bigtable==0.31.1
google-cloud-core==0.28.1
google-cloud-pubsub==0.39.0
googledatastore>=7.0.1,<7.1; python_version < "3.0"
grpcio>=1.8,<2
hdfs>=2.1.0,<3.0.0
httplib2>=0.8,<=0.11.3
mock>=1.0.1,<3.0.0
oauth2client>=2.0.1,<4
proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4
protobuf>=3.5.0.post1,<4
pyarrow>=0.11.1,<0.12.0; python_version >= "3.0" or platform_system != "Windows"
pydot>=1.2.0,<1.3
pytz>=2018.3
pyvcf>=0.6.8,<0.7.0; python_version < "3.0"
pyyaml>=3.12,<4.0.0
typing>=3.6.0,<3.7.0; python_version < "3.5.0"
- -
- -
2.10.0 - -

Beam SDK for Python 2.10.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro-python3>=1.8.1,<2.0.0; python_version >= "3.0"
avro>=1.8.1,<2.0.0; python_version < "3.0"
crcmod>=1.7,<2.0
dill>=0.2.9,<0.2.10
fastavro>=0.21.4,<0.22
future>=0.16.0,<1.0.0
futures>=3.1.1,<4.0.0
google-apitools>=0.5.23,<=0.5.24
google-cloud-bigquery>=1.6.0,<1.7.0
google-cloud-pubsub==0.39.0
googledatastore>=7.0.1,<7.1; python_version < "3.0"
grpcio>=1.8,<2
hdfs>=2.1.0,<3.0.0
httplib2>=0.8,<=0.11.3
mock>=1.0.1,<3.0.0
oauth2client>=2.0.1,<4
proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4
protobuf>=3.5.0.post1,<4
pyarrow>=0.11.1,<0.12.0; python_version >= "3.0" or platform_system != "Windows"
pydot>=1.2.0,<1.3
pytz>=2018.3
pyvcf>=0.6.8,<0.7.0
pyyaml>=3.12,<4.0.0
typing>=3.6.0,<3.7.0; python_version < "3.5.0"
- -
- -
2.9.0 - -

Beam SDK for Python 2.9.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro>=1.8.1, <2.0.0
crcmod>=1.7, <2.0
dill>=0.2.6, <=0.2.8.2
fastavro>=0.21.4, <0.22
future>=0.16.0, <1.0.0
futures>=3.1.1, <4.0.0
google-apitools>=0.5.23, <0.5.24
google-cloud-pubsub==0.35.4
google-cloud-bigquery>=1.6.0, <1.7.0
googledatastore>=7.0.1, <7.1
grpcio>=1.8, <2
hdfs>=2.1.0, <3.0.0
httplib2>=0.8, <=0.11.3
mock>=1.0.1, <3.0.0
nose>=1.3.7
numpy>=1.14.3, <2
oauth2client>=2.0.1, <4
parameterized>=0.6.0, <0.7.0
proto-google-cloud-datastore-v1>=0.90.0, <0.90.4
protobuf>=3.5.0.post1, <4
pydot>=1.2.0, <1.3
pyhamcrest>=1.9, <2.0
pytz>=2018.3, <=2018.4
pyyaml>=3.12, <4.0.0
pyvcf>=0.6.8, <0.7.0
typing>=3.6.0, <3.7.0
- -
- -
2.8.0 - -

Beam SDK for Python 2.8.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro>=1.8.1, <2.0.0
crcmod>=1.7, <2.0
dill>=0.2.6, <=0.2.8.2
fastavro>=0.21.4, <0.22
future>=0.16.0, <1.0.0
futures>=3.1.1, <4.0.0
google-apitools>=0.5.18, <0.5.20
google-cloud-pubsub==0.26.0
google-cloud-bigquery==0.25.0
googledatastore==7.0.1
grpcio>=1.8, <2
hdfs>=2.1.0, <3.0.0
httplib2>=0.8, <=0.11.3
mock>=1.0.1, <3.0.0
nose>=1.3.7
numpy>=1.14.3, <2
oauth2client>=2.0.1, <5
parameterized>=0.6.0, <0.7.0
proto-google-cloud-datastore-v1>=0.90.0, <0.90.4
proto-google-cloud-pubsub-v1==0.15.4
protobuf>=3.5.0.post1, <4
pydot>=1.2.0, <1.3
pyhamcrest>=1.9, <2.0
pytz>=2018.3, <=2018.4
pyyaml>=3.12, <4.0.0
pyvcf>=0.6.8, <0.7.0
typing>=3.6.0, <3.7.0
- -
- -
2.7.0 - -

Beam SDK for Python 2.7.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro>=1.8.1, <2.0.0
crcmod>=1.7, <2.0
dill>=0.2.6, <=0.2.8.2
fastavro==0.19.7
future>=0.16.0, <1.0.0
futures>=3.1.1, <4.0.0
google-apitools>=0.5.18, <0.5.20
google-cloud-pubsub==0.26.0
google-cloud-bigquery==0.25.0
googledatastore==7.0.1
grpcio>=1.8, <2
hdfs>=2.1.0, <3.0.0
httplib2>=0.8, <=0.11.3
mock>=1.0.1, <3.0.0
nose>=1.3.7
numpy>=1.14.3, <2
oauth2client>=2.0.1, <5
proto-google-cloud-datastore-v1>=0.90.0, <0.90.4
proto-google-cloud-pubsub-v1==0.15.4
protobuf>=3.5.0.post1, <4
pydot>=1.2.0, <1.3
pyhamcrest>=1.9, <2.0
pytz>=2018.3, <=2018.4
pyyaml>=3.12, <4.0.0
pyvcf>=0.6.8, <0.7.0
six>=1.9, <1.12
typing>=3.6.0, <3.7.0
- -
- -
2.6.0 - -

Beam SDK for Python 2.6.0 has the following compile and - runtime dependencies.

- - - - - - - - - - - - - - - - - - - - - - - - - - -
PackageVersion
avro>=1.8.1,<2.0.0
crcmod>=1.7,<2.0
dill>=0.2.6,<=0.2.8.2
future>=0.16.0,<1.0.0
futures>=3.1.1,<4.0.0
google-apitools>=0.5.18,<=0.5.20
google-cloud-bigquery==0.25.0
google-cloud-pubsub==0.26.0
googledatastore==7.0.1
grpcio>=1.8,<2
hdfs>=2.1.0,<3.0.0
httplib2>=0.8,<=0.11.3
mock>=1.0.1,<3.0.0
oauth2client>=2.0.1,<5
proto-google-cloud-datastore-v1>=0.90.0,<=0.90.4
proto-google-cloud-pubsub-v1==0.15.4
protobuf>=3.5.0.post1,<4
pydot>=1.2.0,<1.3
pytz>=2018.3,<=2018.4
pyvcf>=0.6.8,<0.7.0
pyyaml>=3.12,<4.0.0
six>=1.9,<1.12
typing>=3.6.0,<3.7.0
- -
+ ``` + $ pip install pipdeptree && pipdeptree -p apache-beam + ```