-
Notifications
You must be signed in to change notification settings - Fork 28.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SPARK-8378] [STREAMING] Add the Python API for Flume
Author: zsxwing <zsxwing@gmail.com> Closes #6830 from zsxwing/flume-python and squashes the following commits: 78dfdac [zsxwing] Fix the compile error in the test code f1bf3c0 [zsxwing] Address TD's comments 0449723 [zsxwing] Add sbt goal streaming-flume-assembly/assembly e93736b [zsxwing] Fix the test case for determine_modules_to_test 9d5821e [zsxwing] Fix pyspark_core dependencies f9ee681 [zsxwing] Merge branch 'master' into flume-python 7a55837 [zsxwing] Add streaming_flume_assembly to run-tests.py b96b0de [zsxwing] Merge branch 'master' into flume-python ce85e83 [zsxwing] Fix incompatible issues for Python 3 01cbb3d [zsxwing] Add import sys 152364c [zsxwing] Fix the issue that StringIO doesn't work in Python 3 14ba0ff [zsxwing] Add flume-assembly for sbt building b8d5551 [zsxwing] Merge branch 'master' into flume-python 4762c34 [zsxwing] Fix the doc 0336579 [zsxwing] Refactor Flume unit tests and also add tests for Python API 9f33873 [zsxwing] Add the Python API for Flume
- Loading branch information
Showing
15 changed files
with
1,009 additions
and
236 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one or more | ||
| # contributor license agreements. See the NOTICE file distributed with | ||
| # this work for additional information regarding copyright ownership. | ||
| # The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| # (the "License"); you may not use this file except in compliance with | ||
| # the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
| # | ||
|
|
||
| """ | ||
| Counts words in UTF8 encoded, '\n' delimited text received from the network every second. | ||
| Usage: flume_wordcount.py <hostname> <port> | ||
| To run this on your local machine, you need to setup Flume first, see | ||
| https://flume.apache.org/documentation.html | ||
| and then run the example | ||
| `$ bin/spark-submit --jars external/flume-assembly/target/scala-*/\ | ||
| spark-streaming-flume-assembly-*.jar examples/src/main/python/streaming/flume_wordcount.py \ | ||
| localhost 12345 | ||
| """ | ||
| from __future__ import print_function | ||
|
|
||
| import sys | ||
|
|
||
| from pyspark import SparkContext | ||
| from pyspark.streaming import StreamingContext | ||
| from pyspark.streaming.flume import FlumeUtils | ||
|
|
||
| if __name__ == "__main__": | ||
| if len(sys.argv) != 3: | ||
| print("Usage: flume_wordcount.py <hostname> <port>", file=sys.stderr) | ||
| exit(-1) | ||
|
|
||
| sc = SparkContext(appName="PythonStreamingFlumeWordCount") | ||
| ssc = StreamingContext(sc, 1) | ||
|
|
||
| hostname, port = sys.argv[1:] | ||
| kvs = FlumeUtils.createStream(ssc, hostname, int(port)) | ||
| lines = kvs.map(lambda x: x[1]) | ||
| counts = lines.flatMap(lambda line: line.split(" ")) \ | ||
| .map(lambda word: (word, 1)) \ | ||
| .reduceByKey(lambda a, b: a+b) | ||
| counts.pprint() | ||
|
|
||
| ssc.start() | ||
| ssc.awaitTermination() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <!-- | ||
| ~ Licensed to the Apache Software Foundation (ASF) under one or more | ||
| ~ contributor license agreements. See the NOTICE file distributed with | ||
| ~ this work for additional information regarding copyright ownership. | ||
| ~ The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| ~ (the "License"); you may not use this file except in compliance with | ||
| ~ the License. You may obtain a copy of the License at | ||
| ~ | ||
| ~ http://www.apache.org/licenses/LICENSE-2.0 | ||
| ~ | ||
| ~ Unless required by applicable law or agreed to in writing, software | ||
| ~ distributed under the License is distributed on an "AS IS" BASIS, | ||
| ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| ~ See the License for the specific language governing permissions and | ||
| ~ limitations under the License. | ||
| --> | ||
|
|
||
| <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | ||
| <modelVersion>4.0.0</modelVersion> | ||
| <parent> | ||
| <groupId>org.apache.spark</groupId> | ||
| <artifactId>spark-parent_2.10</artifactId> | ||
| <version>1.5.0-SNAPSHOT</version> | ||
| <relativePath>../../pom.xml</relativePath> | ||
| </parent> | ||
|
|
||
| <groupId>org.apache.spark</groupId> | ||
| <artifactId>spark-streaming-flume-assembly_2.10</artifactId> | ||
| <packaging>jar</packaging> | ||
| <name>Spark Project External Flume Assembly</name> | ||
| <url>http://spark.apache.org/</url> | ||
|
|
||
| <properties> | ||
| <sbt.project.name>streaming-flume-assembly</sbt.project.name> | ||
| </properties> | ||
|
|
||
| <dependencies> | ||
| <dependency> | ||
| <groupId>org.apache.spark</groupId> | ||
| <artifactId>spark-streaming-flume_${scala.binary.version}</artifactId> | ||
| <version>${project.version}</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.spark</groupId> | ||
| <artifactId>spark-streaming_${scala.binary.version}</artifactId> | ||
| <version>${project.version}</version> | ||
| <scope>provided</scope> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.avro</groupId> | ||
| <artifactId>avro</artifactId> | ||
| <version>${avro.version}</version> | ||
| </dependency> | ||
| <dependency> | ||
| <groupId>org.apache.avro</groupId> | ||
| <artifactId>avro-ipc</artifactId> | ||
| <version>${avro.version}</version> | ||
| <exclusions> | ||
| <exclusion> | ||
| <groupId>io.netty</groupId> | ||
| <artifactId>netty</artifactId> | ||
| </exclusion> | ||
| <exclusion> | ||
| <groupId>org.mortbay.jetty</groupId> | ||
| <artifactId>jetty</artifactId> | ||
| </exclusion> | ||
| <exclusion> | ||
| <groupId>org.mortbay.jetty</groupId> | ||
| <artifactId>jetty-util</artifactId> | ||
| </exclusion> | ||
| <exclusion> | ||
| <groupId>org.mortbay.jetty</groupId> | ||
| <artifactId>servlet-api</artifactId> | ||
| </exclusion> | ||
| <exclusion> | ||
| <groupId>org.apache.velocity</groupId> | ||
| <artifactId>velocity</artifactId> | ||
| </exclusion> | ||
| </exclusions> | ||
| </dependency> | ||
| </dependencies> | ||
|
|
||
| <build> | ||
| <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> | ||
| <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> | ||
| <plugins> | ||
| <plugin> | ||
| <groupId>org.apache.maven.plugins</groupId> | ||
| <artifactId>maven-shade-plugin</artifactId> | ||
| <configuration> | ||
| <shadedArtifactAttached>false</shadedArtifactAttached> | ||
| <outputFile>${project.build.directory}/scala-${scala.binary.version}/spark-streaming-flume-assembly-${project.version}.jar</outputFile> | ||
| <artifactSet> | ||
| <includes> | ||
| <include>*:*</include> | ||
| </includes> | ||
| </artifactSet> | ||
| <filters> | ||
| <filter> | ||
| <artifact>*:*</artifact> | ||
| <excludes> | ||
| <exclude>META-INF/*.SF</exclude> | ||
| <exclude>META-INF/*.DSA</exclude> | ||
| <exclude>META-INF/*.RSA</exclude> | ||
| </excludes> | ||
| </filter> | ||
| </filters> | ||
| </configuration> | ||
| <executions> | ||
| <execution> | ||
| <phase>package</phase> | ||
| <goals> | ||
| <goal>shade</goal> | ||
| </goals> | ||
| <configuration> | ||
| <transformers> | ||
| <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> | ||
| <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> | ||
| <resource>reference.conf</resource> | ||
| </transformer> | ||
| <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> | ||
| <resource>log4j.properties</resource> | ||
| </transformer> | ||
| <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"/> | ||
| <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"/> | ||
| </transformers> | ||
| </configuration> | ||
| </execution> | ||
| </executions> | ||
| </plugin> | ||
| </plugins> | ||
| </build> | ||
| </project> | ||
|
|
Oops, something went wrong.